diff --git a/.github/actions/terragrunt/README.md b/.github/actions/terragrunt/README.md index fbe58fcc..b3f6a2b3 100644 --- a/.github/actions/terragrunt/README.md +++ b/.github/actions/terragrunt/README.md @@ -1,6 +1,6 @@ # Execute Terraform & Terragrunt -This GitHub Action sets up **Terraform** and **Terragrunt** and runs a specified `terragrunt` action: `apply`, `plan`, `apply_plan`, `destroy`, or `init`. When the action needs AWS, the workflow job should configure credentials first. +This GitHub Action sets up **Terraform** and **Terragrunt** and runs a specified `terragrunt` action: `apply`, `plan`, `apply_plan`, `destroy`, `init`, or `graph`. When the action needs AWS, the workflow job should configure credentials first. ## Features @@ -10,7 +10,8 @@ This GitHub Action sets up **Terraform** and **Terragrunt** and runs a specified - Optionally passes Terragrunt variables via JSON tfvars - Supports `plan` mode for producing local saved plan files - Supports `init` mode for outputs-only reads -- Relies on shared Terragrunt root hooks for per-stack saved plan artifact upload and download +- Supports `graph` mode for raw `terragrunt run-all graph-dependencies` output capture +- Writes saved plan files into the live stack directory so workflows can upload and download them with GitHub artifacts - Exports Terragrunt outputs as compact JSON when state exists The Terragrunt install step is kept in this repo-local action rather than hidden behind a third-party Terragrunt wrapper action so the repo can control the exact setup-action revision and react quickly to GitHub Actions runtime deprecations or nested dependency warnings. @@ -24,7 +25,7 @@ The Terragrunt install step is kept in this repo-local action rather than hidden | `aws_region` | AWS region to use | No | `eu-west-2` | | `override_tg_vars` | Terragrunt variables in JSON, written to `override_tg_vars.tfvars.json` | No | `{}` | | `tg_directory` | Directory containing the Terragrunt config | Yes | β€” | -| `tg_action` | Terragrunt action: `apply`, `plan`, `apply_plan`, `destroy`, or `init` | Yes | `apply` | +| `tg_action` | Terragrunt action: `apply`, `plan`, `apply_plan`, `destroy`, `init`, or `graph` | Yes | `apply` | `override_tg_vars` is written for `apply`, `plan`, and `destroy`, but not for `init`. @@ -33,28 +34,31 @@ The Terragrunt install step is kept in this repo-local action rather than hidden | Name | Description | |---|---| | `tg_outputs` | All Terraform outputs in compact JSON. If no state exists, returns `{}` | +| `tg_graph_output` | Raw Terragrunt `run-all graph-dependencies` output. Set only for `tg_action: graph` | ## Behavior - `apply` Runs `terragrunt apply -auto-approve` - `plan` - Runs `terragrunt plan -detailed-exitcode -out=terragrunt.tfplan`. The shared Terragrunt root `after_hook` then renders `terragrunt.plan.txt`, writes `terragrunt.plan.meta.json`, and uploads the per-stack plan bundle to the derived plan bucket when `TG_ENABLE_PLAN_ARTIFACTS=true` and `PLAN_ARTIFACT_RUN_ID` is set. + Runs `terragrunt plan -detailed-exitcode -out=/terragrunt.tfplan`. The action writes `terragrunt.plan.meta.json` into the live stack directory for every plan run, including `has_changes` and `contains_mocked_outputs`, and writes `terragrunt.plan.txt` alongside the binary plan when the plan has changes. - `apply_plan` - Runs `terragrunt apply terragrunt.tfplan`. The shared Terragrunt root `before_hook` downloads the saved plan bundle into the Terragrunt working directory when `TG_ENABLE_PLAN_ARTIFACTS=true` and `PLAN_ARTIFACT_RUN_ID` is set, and fails early if the saved metadata reports mocked dependency outputs. + Runs `terragrunt apply /terragrunt.tfplan`. The calling workflow is expected to download that stack's saved plan artifact into the live stack directory before invoking `apply_plan`. The action also requires `terragrunt.plan.meta.json` to be present there. If that metadata file is missing, or if it says `contains_mocked_outputs: true`, the action fails before apply and tells the operator to regenerate the plan from real upstream outputs. - `destroy` Runs `terragrunt destroy -auto-approve` - `init` Runs `terragrunt init -input=false -reconfigure` and then captures outputs +- `graph` + Runs `terragrunt run-all graph-dependencies --terragrunt-non-interactive --terragrunt-include-external-dependencies --terragrunt-log-level error` and exposes the raw output as `tg_graph_output` ## Saved Plan Layout - One run-level metadata file is stored separately by the shared infra wrapper as a GitHub Actions artifact: - artifact name: `infra-plan-metadata` - - file: `plan-metadata.json` -- Each Terragrunt stack or module stores its own plan bundle at: - - `s3:///terragrunt_plan///terragrunt-plan-/terragrunt.tfplan` - - `s3:///terragrunt_plan///terragrunt-plan-/terragrunt.plan.txt` - - `s3:///terragrunt_plan///terragrunt-plan-/terragrunt.plan.meta.json` + - file: `plan-metadata.json` containing the frozen workflow inputs and derived `waves` +- Each Terragrunt stack or module stores its own plan bundle as a GitHub Actions artifact named `terragrunt-plan--`: + - `terragrunt.plan.meta.json` + - `terragrunt.tfplan` only when changes exist + - `terragrunt.plan.txt` only when changes exist ## AWS Credentials @@ -139,7 +143,7 @@ jobs: tg_action: plan ``` -### Apply From Uploaded Plan In S3 +### Apply From Downloaded GitHub Artifact ```yaml jobs: @@ -165,4 +169,4 @@ jobs: tg_action: apply_plan ``` -This action expects the workflow to set both `TG_ENABLE_PLAN_ARTIFACTS=true` and `PLAN_ARTIFACT_RUN_ID` when using cross-run saved plans so the shared Terragrunt root hooks can resolve the per-stack plan bundle location from the derived plan bucket and environment. +This action expects the workflow to download the matching per-stack plan artifact into the live stack directory before using `tg_action: apply_plan`. diff --git a/.github/actions/terragrunt/action.yml b/.github/actions/terragrunt/action.yml index 274b858e..c0e99163 100644 --- a/.github/actions/terragrunt/action.yml +++ b/.github/actions/terragrunt/action.yml @@ -21,7 +21,7 @@ inputs: description: "Module directory to perform action upon" required: true tg_action: - description: "Terragrunt action to perform (`apply`, `plan`, `apply_plan`, `destroy`, or `init` for outputs-only)" + description: "Terragrunt action to perform (`apply`, `plan`, `apply_plan`, `destroy`, `init` for outputs-only, or `graph` for raw run-all graph output)" required: true default: apply @@ -29,6 +29,9 @@ outputs: tg_outputs: description: "All Terraform outputs in JSON format" value: ${{ steps.tg_outputs.outputs.terraform_json }} + tg_graph_output: + description: "Raw Terragrunt run-all dependency graph output" + value: ${{ steps.tg_graph.outputs.graph_output }} runs: using: "composite" @@ -70,16 +73,20 @@ runs: TG_PLAN_LOG_ABS_PATH: ${{ github.workspace }}/${{ inputs.tg_directory }}/terragrunt.plan.log working-directory: ${{ inputs.tg_directory }} run: | - PLAN_PATH="terragrunt.tfplan" - PLAN_LOG_PATH="$(pwd)/terragrunt.plan.log" + PLAN_DIR="${{ github.workspace }}/${{ inputs.tg_directory }}" + PLAN_PATH="${PLAN_DIR}/terragrunt.tfplan" + PLAN_TEXT_PATH="${PLAN_DIR}/terragrunt.plan.txt" + PLAN_META_PATH="${PLAN_DIR}/terragrunt.plan.meta.json" + PLAN_JSON_PATH="${PLAN_DIR}/terragrunt.plan.json" + PLAN_LOG_PATH="${PLAN_DIR}/terragrunt.plan.log" case "${{ inputs.tg_action }}" in apply) - terragrunt apply -auto-approve -compact-warnings -var-file=override_tg_vars.tfvars.json + terragrunt --terragrunt-non-interactive apply -auto-approve -compact-warnings -var-file=override_tg_vars.tfvars.json ;; plan) set +e - terragrunt plan -input=false -lock=false -detailed-exitcode -compact-warnings -out="$PLAN_PATH" -var-file=override_tg_vars.tfvars.json 2>&1 | tee "$PLAN_LOG_PATH" + terragrunt --terragrunt-non-interactive plan -input=false -lock=false -detailed-exitcode -compact-warnings -out="$PLAN_PATH" -var-file=override_tg_vars.tfvars.json 2>&1 | tee "$PLAN_LOG_PATH" plan_exit_code=${PIPESTATUS[0]} set -e @@ -87,21 +94,55 @@ runs: exit 1 fi + plan_has_changes=false + if [ "$plan_exit_code" -eq 2 ]; then + plan_has_changes=true + fi + plan_contains_mocked_outputs=false if grep -Fq "mock outputs provided and returning those in dependency output" "$PLAN_LOG_PATH"; then plan_contains_mocked_outputs=true echo "::warning title=Mock outputs used during plan::Terragrunt used dependency mock outputs while creating a saved plan. This plan artifact should not be used with apply_plan until a fresh plan is created from real upstream outputs." fi + jq -n \ + --arg tg_directory "${{ inputs.tg_directory }}" \ + --argjson has_changes "$plan_has_changes" \ + --argjson contains_mocked_outputs "$plan_contains_mocked_outputs" \ + '{tg_directory: $tg_directory, has_changes: $has_changes, contains_mocked_outputs: $contains_mocked_outputs}' \ + > "$PLAN_META_PATH" + + echo "=== terragrunt.plan.meta.json ===" + cat "$PLAN_META_PATH" + + terragrunt show -no-color "$PLAN_PATH" > "$PLAN_TEXT_PATH" + echo "plan_exit_code=$plan_exit_code" >> "$GITHUB_OUTPUT" echo "plan_contains_mocked_outputs=$plan_contains_mocked_outputs" >> "$GITHUB_OUTPUT" - echo "Terragrunt binary plan path in cache: $PLAN_PATH" + echo "Terragrunt binary plan path: $PLAN_PATH" ;; apply_plan) + if [ ! -f "$PLAN_PATH" ]; then + echo "::error title=Missing saved plan artifact::Expected '${PLAN_PATH}' to exist before apply_plan." + exit 1 + fi + + if [ ! -f "$PLAN_META_PATH" ]; then + echo "::error title=Missing saved plan metadata::Expected '${PLAN_META_PATH}' to exist before apply_plan." + exit 1 + fi + + plan_contains_mocked_outputs="$(jq -r '.contains_mocked_outputs // false' "$PLAN_META_PATH")" + + if [ "$plan_contains_mocked_outputs" = "true" ]; then + echo "::error title=Saved plan contains mocked outputs::Saved plan metadata indicates mocked outputs were used. Regenerate it after upstream real outputs exist." + exit 1 + fi + set +e APPLY_LOG_PATH="$(pwd)/terragrunt.apply.log" - terragrunt apply -auto-approve "$PLAN_PATH" 2>&1 | tee "$APPLY_LOG_PATH" + terragrunt --terragrunt-non-interactive apply -auto-approve "$PLAN_PATH" 2>&1 | tee "$APPLY_LOG_PATH" apply_exit_code=${PIPESTATUS[0]} set -e @@ -123,33 +164,41 @@ runs: emit_error "Saved plan is stale" "Saved plan is stale" fi - emit_error "Saved plan artifact missing" "Saved plan artifact not found for" - emit_error "Missing PLAN_ARTIFACT_RUN_ID" "PLAN_ARTIFACT_RUN_ID is required when TG_ENABLE_PLAN_ARTIFACTS=true" - - if grep -Fq "Plan bucket" "$APPLY_LOG_PATH" && grep -Fq "does not exist" "$APPLY_LOG_PATH"; then - emit_error "Plan bucket missing" "Plan bucket" - fi - - emit_error "Plan bucket creation declined" "Plan bucket creation declined." - emit_error "Plan bucket confirmation unavailable" "no interactive terminal is available for confirmation" exit "$apply_exit_code" fi ;; destroy) - terragrunt destroy -auto-approve -compact-warnings -var-file=override_tg_vars.tfvars.json + terragrunt --terragrunt-non-interactive destroy -auto-approve -compact-warnings -var-file=override_tg_vars.tfvars.json ;; init) echo "Running init only (no infra changes)..." - terragrunt init -input=false -reconfigure + terragrunt --terragrunt-non-interactive init -input=false -reconfigure + ;; + graph) + echo "Graph mode runs in the dedicated graph step." ;; *) - echo "Unknown tg_action: '${{ inputs.tg_action }}' (expected: apply|plan|apply_plan|destroy|init)" >&2 + echo "Unknown tg_action: '${{ inputs.tg_action }}' (expected: apply|plan|apply_plan|destroy|init|graph)" >&2 exit 2 ;; esac + - name: Capture Terragrunt graph JSON + if: inputs.tg_action == 'graph' + id: tg_graph + shell: bash + working-directory: ${{ inputs.tg_directory }} + run: | + echo "πŸ•ΈοΈ Rendering Terragrunt run-all dependency graph..." + { + echo "graph_output<> "$GITHUB_OUTPUT" + echo "βœ… Terragrunt graph captured." + - name: Capture Terraform Outputs - if: inputs.tg_action != 'destroy' + if: inputs.tg_action != 'destroy' && inputs.tg_action != 'graph' id: tg_outputs shell: bash working-directory: ${{ inputs.tg_directory }} diff --git a/.github/docs/README.md b/.github/docs/README.md index 952ffc81..e1a41817 100644 --- a/.github/docs/README.md +++ b/.github/docs/README.md @@ -19,8 +19,8 @@ Use it when you need to understand: - Release and validation: `release.yml`, `pull_request.yml` - Shared artifact prep and build: `shared_infra_releases.yml`, `shared_build.yml`, `shared_build_get.yml` -- Shared infra and code rollout: `shared_infra_plan.yml`, `shared_infra_apply.yml`, `shared_infra_apply_from_plan.yml`, `shared_infra.yml`, `shared_deploy.yml`, `shared_directories_get.yml` -- Environment entry points: `dev_infra_apply.yml`, `dev_infra_plan.yml`, `dev_infra_plan_and_apply.yml`, `dev_infra_apply_from_plan.yml`, `dev_code_deploy.yml`, `prod_infra_apply.yml`, `prod_infra_plan.yml`, `prod_infra_apply_from_plan.yml` +- Shared infra and code rollout: `shared_infra_plan.yml`, `shared_infra_apply_no_plan.yml`, `shared_infra_apply_from_plan.yml`, `shared_infra.yml`, `shared_deploy.yml`, `shared_directories_get.yml` +- Environment entry points: `dev_infra_apply_no_plan.yml`, `dev_infra_plan.yml`, `dev_infra_plan_and_apply.yml`, `dev_infra_apply_from_plan.yml`, `dev_code_deploy.yml`, `prod_infra_apply_no_plan.yml`, `prod_infra_plan.yml`, `prod_infra_apply_from_plan.yml` - Cleanup: `destroy.yml` ## Workflow Contracts @@ -82,14 +82,18 @@ flowchart LR ### Infra And Code Rollout - `shared_infra_plan.yml` - Plan wrapper around `shared_infra.yml`. It takes resolved workflow inputs directly, uploads one run-level `plan-metadata.json` file as a GitHub Actions artifact named `infra-plan-metadata`, and then calls `shared_infra.yml` with `tg_action: plan` plus `plan_run_id: ${{ github.run_id }}`. After the plan completes, it prints the current workflow `github.run_id` into both the logs and the GitHub Actions step summary as `plan_artifact_run_id`, and exposes that value as a reusable-workflow output. -- `shared_infra_apply.yml` + Plan wrapper around `shared_infra.yml`. It takes resolved workflow inputs directly, starts `shared_get_modules.yml` to derive the current wave outputs, writes those waves plus the direct workflow inputs into one run-level `plan-metadata.json` file, uploads that file as a GitHub Actions artifact named `infra-plan-metadata`, and then calls `shared_infra.yml` with `tg_action: plan` plus `plan_run_id: ${{ github.run_id }}`. After the plan completes, it prints the current workflow `github.run_id` into both the logs and the GitHub Actions step summary as `plan_artifact_run_id`, and exposes that value as a reusable-workflow output. +- `shared_infra_apply_no_plan.yml` Direct-input apply wrapper around `shared_infra.yml`. It takes resolved workflow inputs directly and calls `shared_infra.yml` with `tg_action: apply`. - `shared_infra_apply_from_plan.yml` - Apply-from-plan wrapper around `shared_infra.yml`. It takes `plan_artifact_run_id`, downloads the `infra-plan-metadata` GitHub artifact from that earlier workflow run, reads the frozen graph inputs back out, and then calls `shared_infra.yml` with `tg_action: apply_plan` plus `plan_run_id: `. Per-stack plan bundle download still happens inside the shared Terragrunt root `before_hook`. + Apply-from-plan wrapper around `shared_infra.yml`. It takes `plan_artifact_run_id`, downloads the `infra-plan-metadata` GitHub artifact from that earlier workflow run, reads the frozen graph inputs and saved wave arrays back out, and then reruns the same `wave_0`, `wave_1`, `wave_2`, and `wave_3` module order. Each per-module job downloads its matching `terragrunt-plan--` GitHub artifact into the live stack directory and then invokes the repo-local Terragrunt action with `tg_action: apply_plan`. - `shared_infra.yml` - Pure ordered infra graph executor. It applies shared stacks first, then runtime stacks, then frontend infrastructure. Shared stacks now include the CloudWatch observability dashboard. It accepts `tg_action` so the same graph can run a normal apply, upload derived per-stack plan artifacts to the dedicated plan bucket under `terragrunt_plan/`, or apply from previously uploaded plan artifacts. The wrapper workflows now pass a single `plan_run_id`, exported to Terragrunt jobs as `PLAN_ARTIFACT_RUN_ID`, while each Terragrunt job configures AWS credentials at job start and then reuses that ambient session in the repo-local Terragrunt action. That means each infra run has one shared run-level metadata artifact (`infra-plan-metadata`) for the whole graph and one separate saved plan bundle per Terragrunt stack or module. Saved-plan transfer is opt-in: the shared workflow sets `TG_ENABLE_PLAN_ARTIFACTS=true` only for `plan` and `apply_plan`. In `plan` mode, the shared Terragrunt root `after_hook` renders and uploads each per-stack plan bundle. In `apply_plan` mode, the shared Terragrunt root `before_hook` downloads the saved plan bundle before `terragrunt apply` runs and fails if the saved metadata says mocked outputs were used. Its visible step labels now follow the high-level operation, so both direct apply and apply-from-plan render as `Apply` while plan still renders as `Plan`. Bootstrap-sensitive edges such as `security -> network` should be modeled with Terragrunt `dependency` blocks plus constrained `mock_outputs` in the live stack so `plan` and `validate` can run before upstream state exists, while `apply` still resolves real outputs. + Temporary wave-based executor. It delegates graph and wave discovery to `shared_get_modules.yml`, exposes the resulting `waves_json` as the reusable-workflow output, and then runs `wave_0`, `wave_1`, `wave_2`, and `wave_3` jobs in dependency order. Each wave only runs when its module array is non-empty, fans that array out as a matrix, checks out the requested ref, configures AWS credentials once per matrix job, and invokes the shared repo-local Terragrunt action against `infra/live//aws/`. The deprecated `changed_items_json` workflow output is still present for compatibility and currently mirrors `waves_json`. +- `just --justfile justfile.ci tg-graph-json-to-waves` + CI helper that expects compact graph JSON in `TG_GRAPH_JSON` and returns a sequential JSON array of wave objects like `[{ "wave": 0, "modules": [...] }, ...]`, with each wave containing only modules whose direct dependencies were satisfied by earlier waves. - The shared infra wrappers must forward the permissions required by the nested reusable call chain. In practice that means `id-token: write` everywhere the Terragrunt action may assume AWS OIDC and `contents: read` for checkout. The shared plan/apply wrappers now rely on AWS access to the shared code bucket rather than GitHub artifact permissions for cross-run recovery. +- The shared infra wrappers no longer accept `lambda_matrix` or `service_matrix`. Infra selection now comes from the Terragrunt dependency graph and derived waves, not from precomputed runtime directory matrices. +- The shared infra wrappers no longer accept `code_bucket` either. The current graph-wave placeholder path only needs `environment`, `infra_version`, optional `bootstrap_image_uri`, and the Terragrunt action context. - `shared_deploy.yml` Rolls out Lambda code, optional migrations, optional reconciliation Lambdas, ECS task and service updates, and optional frontend deploys. Its multi-step AWS jobs now configure credentials once at job start and let the local `just` and Terragrunt actions reuse that ambient session. The reusable workflow renders its Lambda and ECS CodeDeploy AppSpec files from the shared templates under `config/deploy/`, and its mutating `just` steps should target `justfile.deploy` rather than the repo-root `justfile`. @@ -106,15 +110,15 @@ flowchart LR ### Wrapper Workflows -- `dev_infra_apply.yml` - Entry point for dev infra apply. +- `dev_infra_apply_no_plan.yml` + Entry point for dev infra apply. It currently calls the shared infra workflow directly with an empty placeholder `bootstrap_image_uri`, because the temporary wave-placeholder executor does not yet consume that old artifact input. - `dev_infra_plan.yml` - Entry point for dev infra plan. It discovers runtime directories, prepares dev artifact references, and then runs the shared infra wrapper in direct-input `plan` mode. + Entry point for dev infra plan. It currently calls the shared infra plan wrapper directly with an empty placeholder `bootstrap_image_uri`, because the temporary wave-placeholder executor does not yet consume that old artifact input. - `dev_infra_plan_and_apply.yml` Entry point for dev infra plan-then-apply. It captures the current workflow `run_id` as plan context, runs the shared infra wrapper in direct-input `plan` mode so that the wrapper emits both plan artifacts and `infra-plan-metadata`, and then reruns the same ordered infra graph in metadata-backed `apply_plan` mode. - `prod_infra_plan.yml` Entry point for prod infra plan. It resolves released artifacts from `ci` and then runs the shared infra wrapper in direct-input `plan` mode so that it emits both the reusable metadata artifact and the derived per-stack plan artifacts for that resolved input set. -- `prod_infra_apply.yml` +- `prod_infra_apply_no_plan.yml` Entry point for prod infra apply using shared artifacts from `ci`. - `dev_infra_apply_from_plan.yml` Entry point for dev infra apply-from-plan. It takes a prior `plan_artifact_run_id` from an earlier `dev_infra_plan.yml` or `dev_infra_plan_and_apply.yml` run and reruns the ordered dev infra graph through `shared_infra_apply_from_plan.yml`. @@ -128,10 +132,12 @@ flowchart LR ### Cleanup And Discovery - `destroy.yml` - Tears down app layers before shared dependencies, including the shared observability dashboard and any environment-owned shared artifact stacks such as the `dev` code bucket. The workflow-dispatch input `allow_prod_cleanup` now gates every cleanup or destroy job that is normally skipped for `prod`, including the `Code Bucket`, `ECR`, and final tagged-resource cleanup jobs. After the main graph completes, the workflow first counts tagged leftovers through `justfile.destroy`, prints a warning only when any remain, and then runs the cleanup recipe. That cleanup currently deletes leaked Cognito user pools, deregisters and then deletes leaked ECS task-definition revisions, deletes leftover ECS clusters, and force-deletes leftover Secrets Manager secrets, then validates the remaining tagged ARNs against the underlying service APIs rather than treating the tagging index as the source of truth. Already-removed Cognito pools, ECS task-definition revisions, ECS clusters, or Secrets Manager secrets are treated as successful no-ops so stale tagging API results do not fail cleanup. If unsupported or still-live tagged resources remain after that sweep, the workflow now records a warning and step summary instead of failing the whole destroy run. `prod` runs that same path only when `allow_prod_cleanup` is enabled, and the workflow prints a conspicuous warning first. + Tears down infrastructure through the same Terragrunt graph contract as plan/apply, but in reverse shared-infra wave order. It derives the current module waves through `shared_get_modules.yml`, using that reusable workflow's filtering inputs to omit `oidc` entirely and to omit `code_bucket` and `ecr` from the destroy set for `prod` unless `allow_prod_cleanup` is enabled, and then runs `wave_2`, `wave_1`, and `wave_0`. The dedicated `wave_3` destroy stage is intentionally omitted. The only remaining module-specific destroy placeholder vars are the required ECS task image inputs for `task_*`; the frontend, service, and database stacks now rely on Terragrunt dependency mocks instead of workflow-injected destroy placeholders. After the main graph completes, the workflow first counts tagged leftovers through `justfile.destroy`, prints a warning only when any remain, and then runs the cleanup recipe. That cleanup currently deletes leaked Cognito user pools, deregisters and then deletes leaked ECS task-definition revisions, deletes leftover ECS clusters, and force-deletes leftover Secrets Manager secrets, then validates the remaining tagged ARNs against the underlying service APIs rather than treating the tagging index as the source of truth. Already-removed Cognito pools, ECS task-definition revisions, ECS clusters, or Secrets Manager secrets are treated as successful no-ops so stale tagging API results do not fail cleanup. If unsupported or still-live tagged resources remain after that sweep, the workflow now records a warning and step summary instead of failing the whole destroy run. `prod` runs that same path only when `allow_prod_cleanup` is enabled, and the workflow prints a conspicuous warning first. - `shared_directories_get.yml` Derives the directory-based matrices used by wrapper workflows and PR action-test discovery. The distinction between `service_dirs` and `container_dirs` is intentional: `service_dirs` contains deployable ECS service image directories only, while `container_dirs` also includes shared ECS sidecar image targets such as `debug` and `otel_collector`. ECS artifact builds that feed `shared_build.yml` should use `container_dirs` for `ecs_matrix`, because ECS task deploys need the shared sidecar images as well as the service images. Workflows that only need app service names or task/service stack derivation should use `service_dirs`. +- `shared_get_modules.yml` + Reusable module-discovery workflow for infra waves. It renders the Terragrunt graph for the target environment, converts that graph into compact JSON, derives dependency-safe waves, and exposes `waves_json`, `wave_0_modules`, `wave_1_modules`, `wave_2_modules`, and `wave_3_modules` as reusable-workflow outputs. Callers can pass `ignore_task_modules: true` to exclude any `task_*` modules from the emitted rollout waves for the current bootstrap-oriented infra path, `ignore_shared_artifact_modules: true` to omit shared artifact stacks such as `code_bucket` and `ecr`, and `ignore_oidc_module: true` to exclude `oidc` entirely. ## Feasibility Checks @@ -142,17 +148,17 @@ Run these checks on every CI, workflow, or deploy-contract change. - compare every caller `with:` block against the callee `workflow_call.inputs` - compare expected outputs against actual `jobs..outputs.*` - verify optional inputs are intentionally omitted, not accidentally missing -- the repo-local `./.github/actions/terragrunt` action supports `tg_action: plan` for producing the binary plan locally; the shared Terragrunt root `after_hook` then renders `terragrunt.plan.txt` and writes `terragrunt.plan.meta.json` -- shared Terragrunt root hooks now upload per-stack plan artifacts on `plan` and download them on `apply_plan` only when `TG_ENABLE_PLAN_ARTIFACTS=true`, using the caller-provided `PLAN_ARTIFACT_RUN_ID` plus the root-derived `plan_bucket`, so graph executors like `shared_infra.yml` do not need separate `./.github/actions/just` steps for those transfers +- the repo-local `./.github/actions/terragrunt` action supports `tg_action: plan` for producing the binary plan in the live stack directory, writes `terragrunt.plan.meta.json` there for every saved plan including `has_changes` and `contains_mocked_outputs`, and writes `terragrunt.plan.txt` alongside the binary plan when the plan has changes +- `apply_plan` now expects the calling workflow job to download the matching per-stack GitHub artifact into the live stack directory before invoking Terragrunt, including `terragrunt.plan.meta.json`, and it fails immediately if that metadata file is missing or says `contains_mocked_outputs: true` +- when a live Terragrunt `dependency` block uses `mock_outputs` for planability or destroy safety, default it to `mock_outputs_merge_strategy_with_state = "shallow"` so partial real upstream state does not suppress missing mock keys - both repo-local composite actions, `./.github/actions/just` and `./.github/actions/terragrunt`, now assume AWS credentials are already configured in the current job when they need AWS access. The repo pattern is to run `aws-actions/configure-aws-credentials` at the top of each AWS-using job and then call the local actions without extra auth inputs - `./.github/actions/just` installs the requested `just` version through `extractions/setup-crate@v2` in the same minimal composite-action shape as `extractions/setup-just`, rather than depending on `extractions/setup-just` itself - `./.github/actions/terragrunt` installs the requested Terragrunt version through `jdx/mise-action@v4`, while Terraform stays pinned separately through `hashicorp/setup-terraform` - saved infra-plan storage is intentionally split into two levels: - one run-level metadata artifact named `infra-plan-metadata` containing `plan-metadata.json` - - one per-stack plan bundle under `s3:///terragrunt_plan///terragrunt-plan-/` -- the dedicated plan bucket is repo-wide, derived as `---tfplan`, and plan uniqueness comes from `terragrunt_plan///...` + - one per-stack GitHub artifact named `terragrunt-plan--` - `./.github/actions/terragrunt` derives its plan artifact name from `tg_directory`, so callers do not need to pass artifact naming inputs -- if `apply_plan` is used across separate workflow runs, pass the earlier workflow `run_id` through `plan_artifact_run_id`; the shared wrappers recover both metadata and per-stack plan files from the dedicated plan bucket under `terragrunt_plan///...` +- if `apply_plan` is used across separate workflow runs, pass the earlier workflow `run_id` through `plan_artifact_run_id`; the shared wrappers recover both metadata and per-stack plan files from GitHub artifacts in that earlier run - if a cross-run apply should not ask the operator to re-enter versions or recompute artifact resolution, store both the input versions and the resolved reusable-workflow outputs in a metadata artifact during plan and recover them in the apply wrapper from the earlier `run_id` - keep `shared_infra.yml` as the pure graph executor and prefer handling metadata creation/recovery in the dedicated plan/apply wrappers - when using `./.github/actions/just`, check whether the caller needs the repo-root `justfile` or an explicit `justfile_path` @@ -216,7 +222,7 @@ Run these checks on every CI, workflow, or deploy-contract change. These are the workflows most users trigger directly. -- `dev_infra_apply.yml` +- `dev_infra_apply_no_plan.yml` Discovers directories, prepares dev artifacts, and applies dev infrastructure. - `dev_infra_plan.yml` Discovers directories, prepares dev artifacts, and plans the ordered dev infra graph through `shared_infra_plan.yml`. @@ -226,10 +232,10 @@ These are the workflows most users trigger directly. Reapplies the ordered dev infra graph from plan artifacts created by an earlier dev plan run, using the shared `plan_artifact_run_id` contract end-to-end. - `prod_infra_plan.yml` Resolves released artifacts from `ci`, then plans the ordered prod infra graph so that shared infra emits both the metadata artifact and the derived per-stack plan artifacts. -- `prod_infra_apply.yml` +- `prod_infra_apply_no_plan.yml` Resolves released artifacts from `ci` and applies prod infrastructure. - `prod_infra_apply_from_plan.yml` - Reapplies the ordered prod infra graph from plan artifacts created by a prior `prod_infra_plan` run, using the shared `plan_artifact_run_id` contract end-to-end. `shared_infra_apply_from_plan.yml` reads the matching metadata artifact before delegating to `shared_infra.yml`. + Reapplies the ordered prod infra graph from plan artifacts created by a prior `prod_infra_plan` run, using the shared `plan_artifact_run_id` contract end-to-end. `shared_infra_apply_from_plan.yml` reads the matching metadata artifact first, then each apply job downloads its matching per-stack GitHub artifact before invoking `apply_plan`. - `dev_code_deploy.yml` Discovers directories, builds fresh dev artifacts, resolves deploy inputs, and deploys code to dev. - `prod_code_deploy.yml` diff --git a/.github/workflows/destroy.yml b/.github/workflows/destroy.yml index 0ef17300..740bf3fd 100644 --- a/.github/workflows/destroy.yml +++ b/.github/workflows/destroy.yml @@ -23,45 +23,35 @@ concurrency: # only run destroy when no other deploy/destroy is running for the permissions: id-token: write contents: write + actions: read env: TF_VAR_lambda_version: this + TF_VAR_image_uri: destroy-placeholder + TF_VAR_aws_otel_collector_image_uri: destroy-placeholder + TF_VAR_debug_image_uri: destroy-placeholder AWS_OIDC_ROLE_ARN: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/${{ vars.PROJECT_NAME }}-${{ inputs.environment }}-github-oidc-role AWS_REGION: ${{ vars.AWS_REGION }} - DOMAIN_NAME: ${{ vars.DOMAIN_NAME }} jobs: - setup: - name: Discover - uses: ./.github/workflows/shared_directories_get.yml - - observability: - name: Observability + generate_waves: + name: Discover Waves + uses: ./.github/workflows/shared_get_modules.yml + with: + environment: ${{ inputs.environment }} + infra_version: ${{ github.sha }} + ignore_shared_artifact_modules: true + ignore_oidc_module: true + + wave_2: + name: 2 / ${{ matrix.module }} + needs: generate_waves + if: ${{ needs.generate_waves.outputs.wave_2_modules != '[]' }} runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - - uses: aws-actions/configure-aws-credentials@v6 - with: - role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }} - aws-region: ${{ env.AWS_REGION }} - - - name: Destroy observability infra - uses: ./.github/actions/terragrunt - with: - tg_directory: infra/live/${{ inputs.environment }}/aws/observability - tg_action: destroy - - lambdas: - name: Lambdas - runs-on: ubuntu-latest - needs: - - setup - - services strategy: fail-fast: false matrix: - value: ${{ fromJson(needs.setup.outputs.lambda_dirs) }} + module: ${{ fromJson(needs.generate_waves.outputs.wave_2_modules) }} steps: - uses: actions/checkout@v6 @@ -70,66 +60,23 @@ jobs: role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }} aws-region: ${{ env.AWS_REGION }} - - name: Deploy ${{ matrix.value }} infra + - name: Destroy ${{ matrix.module }} infra uses: ./.github/actions/terragrunt with: - tg_directory: infra/live/${{ inputs.environment }}/aws/${{ matrix.value }} + tg_directory: infra/live/${{ inputs.environment }}/aws/${{ matrix.module }} tg_action: destroy - frontend: - name: Frontend - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - - uses: aws-actions/configure-aws-credentials@v6 - with: - role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }} - aws-region: ${{ env.AWS_REGION }} - - - name: Destroy frontend infra - uses: ./.github/actions/terragrunt - env: - TF_VAR_api_invoke_url: "https://placeholder.execute-api.us-east-1.amazonaws.com" - TF_VAR_domain_name: ${{ env.DOMAIN_NAME }} - TF_VAR_auth_user_pool_id: "destroy-placeholder" - TF_VAR_auth_user_pool_client_id: "destroy-placeholder" - TF_VAR_auth_hosted_ui_url: "https://destroy-placeholder" - TF_VAR_auth_readonly_group_name: "readonly" - with: - tg_directory: infra/live/${{ inputs.environment }}/aws/frontend - tg_action: destroy - - cognito: - name: Cognito - runs-on: ubuntu-latest + wave_1: + name: 1 / ${{ matrix.module }} needs: - - frontend - - network - steps: - - uses: actions/checkout@v6 - - - uses: aws-actions/configure-aws-credentials@v6 - with: - role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }} - aws-region: ${{ env.AWS_REGION }} - - - name: Destroy cognito infra - uses: ./.github/actions/terragrunt - env: - TF_VAR_domain_name: ${{ env.DOMAIN_NAME }} - with: - tg_directory: infra/live/${{ inputs.environment }}/aws/cognito - tg_action: destroy - - services: - name: Services + - generate_waves + - wave_2 + if: ${{ always() && needs.generate_waves.outputs.wave_1_modules != '[]' && (needs.wave_2.result == 'success' || needs.wave_2.result == 'skipped') }} runs-on: ubuntu-latest - needs: setup strategy: fail-fast: false matrix: - value: ${{ fromJson(needs.setup.outputs.ecs_service_dirs) }} + module: ${{ fromJson(needs.generate_waves.outputs.wave_1_modules) }} steps: - uses: actions/checkout@v6 @@ -138,176 +85,23 @@ jobs: role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }} aws-region: ${{ env.AWS_REGION }} - - name: Destroy ${{ matrix.value }} infra + - name: Destroy ${{ matrix.module }} infra uses: ./.github/actions/terragrunt - env: - TF_VAR_bootstrap: "true" - TF_VAR_bootstrap_image_uri: "destroy-placeholder" with: - tg_directory: infra/live/${{ inputs.environment }}/aws/${{ matrix.value }} + tg_directory: infra/live/${{ inputs.environment }}/aws/${{ matrix.module }} tg_action: destroy - tasks: - name: Tasks - runs-on: ubuntu-latest + wave_0: + name: 0 / ${{ matrix.module }} needs: - - setup - - services + - generate_waves + - wave_1 + if: ${{ always() && needs.generate_waves.outputs.wave_0_modules != '[]' && (needs.wave_1.result == 'success' || needs.wave_1.result == 'skipped') }} + runs-on: ubuntu-latest strategy: fail-fast: false matrix: - value: ${{ fromJson(needs.setup.outputs.task_dirs) }} - steps: - - uses: actions/checkout@v6 - - - uses: aws-actions/configure-aws-credentials@v6 - with: - role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }} - aws-region: ${{ env.AWS_REGION }} - - - name: Destroy ${{ matrix.value }} infra - uses: ./.github/actions/terragrunt - env: - TF_VAR_image_uri: "destroy-placeholder" - TF_VAR_debug_image_uri: "destroy-placeholder" - TF_VAR_aws_otel_collector_image_uri: "destroy-placeholder" - with: - tg_directory: infra/live/${{ inputs.environment }}/aws/${{ matrix.value }} - tg_action: destroy - - messaging: - name: Messaging - runs-on: ubuntu-latest - needs: - - lambdas - - services - - tasks - steps: - - uses: actions/checkout@v6 - - - uses: aws-actions/configure-aws-credentials@v6 - with: - role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }} - aws-region: ${{ env.AWS_REGION }} - - - name: Destroy messaging infra - uses: ./.github/actions/terragrunt - with: - tg_directory: infra/live/${{ inputs.environment }}/aws/messaging - tg_action: destroy - - database: - name: Database - runs-on: ubuntu-latest - needs: - - lambdas - - services - - tasks - steps: - - uses: actions/checkout@v6 - - - uses: aws-actions/configure-aws-credentials@v6 - with: - role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }} - aws-region: ${{ env.AWS_REGION }} - - - name: Destroy database infra - uses: ./.github/actions/terragrunt - env: - TF_VAR_database_security_group_id: "destroy-placeholder" - with: - tg_directory: infra/live/${{ inputs.environment }}/aws/database - tg_action: destroy - - network: - name: Network - needs: - - frontend - - services - - tasks - - database - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - - uses: aws-actions/configure-aws-credentials@v6 - with: - role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }} - aws-region: ${{ env.AWS_REGION }} - - - name: Destroy network infra - uses: ./.github/actions/terragrunt - with: - tg_directory: infra/live/${{ inputs.environment }}/aws/network - tg_action: destroy - - security: - name: Security - needs: - - network - - lambdas - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - - uses: aws-actions/configure-aws-credentials@v6 - with: - role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }} - aws-region: ${{ env.AWS_REGION }} - - - name: Destroy security infra - uses: ./.github/actions/terragrunt - with: - tg_directory: infra/live/${{ inputs.environment }}/aws/security - tg_action: destroy - - build-bucket: - name: Code Bucket - if: inputs.environment != 'prod' || inputs.allow_prod_cleanup - needs: - - lambdas - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - - uses: aws-actions/configure-aws-credentials@v6 - with: - role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }} - aws-region: ${{ env.AWS_REGION }} - - - name: Destroy code - uses: ./.github/actions/terragrunt - with: - tg_directory: infra/live/${{ inputs.environment }}/aws/code_bucket - tg_action: destroy - - ecr: - name: ECR - if: inputs.environment != 'prod' || inputs.allow_prod_cleanup - needs: - - network - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - - uses: aws-actions/configure-aws-credentials@v6 - with: - role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }} - aws-region: ${{ env.AWS_REGION }} - - - name: Destroy code - uses: ./.github/actions/terragrunt - with: - tg_directory: infra/live/${{ inputs.environment }}/aws/ecr - tg_action: destroy - - cluster: - name: Cluster - needs: - - services - - tasks - - database - runs-on: ubuntu-latest + module: ${{ fromJson(needs.generate_waves.outputs.wave_0_modules) }} steps: - uses: actions/checkout@v6 @@ -316,22 +110,17 @@ jobs: role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }} aws-region: ${{ env.AWS_REGION }} - - name: Destroy cluster infra + - name: Destroy ${{ matrix.module }} infra uses: ./.github/actions/terragrunt with: - tg_directory: infra/live/${{ inputs.environment }}/aws/cluster + tg_directory: infra/live/${{ inputs.environment }}/aws/${{ matrix.module }} tg_action: destroy cleanup: name: Cleanup if: inputs.environment != 'prod' || inputs.allow_prod_cleanup needs: - - observability - - cognito - - security - - build-bucket - - ecr - - cluster + - wave_0 runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 diff --git a/.github/workflows/dev_infra_apply.yml b/.github/workflows/dev_infra_apply.yml deleted file mode 100644 index 1c088d5d..00000000 --- a/.github/workflows/dev_infra_apply.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: Dev Infra Apply (no plan) - -on: - workflow_dispatch: - -permissions: - id-token: write - contents: write - actions: read - -jobs: - setup: - name: Discover - uses: ./.github/workflows/shared_directories_get.yml - - code: - name: Artifacts - uses: ./.github/workflows/shared_infra_releases.yml - with: - environment: dev - infra_version: ${{ github.sha }} - - infra: - name: Apply - needs: - - setup - - code - uses: ./.github/workflows/shared_infra_apply.yml - with: - environment: dev - infra_version: ${{ github.sha }} - code_bucket: ${{ needs.code.outputs.code_bucket }} - lambda_matrix: ${{ needs.setup.outputs.lambda_dirs }} - bootstrap_image_uri: ${{ needs.code.outputs.bootstrap_image_uri }} - service_matrix: ${{ needs.setup.outputs.ecs_service_dirs }} diff --git a/.github/workflows/dev_infra_apply_no_plan.yml b/.github/workflows/dev_infra_apply_no_plan.yml new file mode 100644 index 00000000..4af79338 --- /dev/null +++ b/.github/workflows/dev_infra_apply_no_plan.yml @@ -0,0 +1,18 @@ +name: Dev Infra Apply (no plan) + +on: + workflow_dispatch: + +permissions: + id-token: write + contents: write + actions: read + +jobs: + infra: + name: Apply + uses: ./.github/workflows/shared_infra_apply_no_plan.yml + with: + environment: dev + infra_version: ${{ github.sha }} + bootstrap_image_uri: "" diff --git a/.github/workflows/dev_infra_plan.yml b/.github/workflows/dev_infra_plan.yml index d50eb069..43e87210 100644 --- a/.github/workflows/dev_infra_plan.yml +++ b/.github/workflows/dev_infra_plan.yml @@ -9,27 +9,10 @@ permissions: actions: read jobs: - setup: - name: Discover - uses: ./.github/workflows/shared_directories_get.yml - - code: - name: Artifacts - uses: ./.github/workflows/shared_infra_releases.yml - with: - environment: dev - infra_version: ${{ github.sha }} - infra: name: Plan - needs: - - setup - - code uses: ./.github/workflows/shared_infra_plan.yml with: environment: dev infra_version: ${{ github.sha }} - code_bucket: ${{ needs.code.outputs.code_bucket }} - lambda_matrix: ${{ needs.setup.outputs.lambda_dirs }} - bootstrap_image_uri: ${{ needs.code.outputs.bootstrap_image_uri }} - service_matrix: ${{ needs.setup.outputs.ecs_service_dirs }} + bootstrap_image_uri: "" diff --git a/.github/workflows/prod_infra_apply.yml b/.github/workflows/prod_infra_apply_no_plan.yml similarity index 67% rename from .github/workflows/prod_infra_apply.yml rename to .github/workflows/prod_infra_apply_no_plan.yml index 3fcbd64a..24d7a7c3 100644 --- a/.github/workflows/prod_infra_apply.yml +++ b/.github/workflows/prod_infra_apply_no_plan.yml @@ -22,11 +22,8 @@ jobs: name: Apply needs: - get_build - uses: ./.github/workflows/shared_infra_apply.yml + uses: ./.github/workflows/shared_infra_apply_no_plan.yml with: environment: prod infra_version: 0.19.13 - code_bucket: ${{ needs.get_build.outputs.code_bucket }} - lambda_matrix: ${{ needs.get_build.outputs.lambda_version_files }} bootstrap_image_uri: ${{ needs.get_build.outputs.bootstrap_image_uri }} - service_matrix: ${{ needs.get_build.outputs.ecs_service_matrix }} diff --git a/.github/workflows/prod_infra_plan.yml b/.github/workflows/prod_infra_plan.yml index 35f94ea8..6170eea9 100644 --- a/.github/workflows/prod_infra_plan.yml +++ b/.github/workflows/prod_infra_plan.yml @@ -47,7 +47,4 @@ jobs: with: environment: prod infra_version: ${{ inputs.infra_version }} - code_bucket: ${{ needs.get_build.outputs.code_bucket }} - lambda_matrix: ${{ needs.get_build.outputs.lambda_version_files }} bootstrap_image_uri: ${{ needs.get_build.outputs.bootstrap_image_uri }} - service_matrix: ${{ needs.get_build.outputs.ecs_service_matrix }} diff --git a/.github/workflows/shared_get_modules.yml b/.github/workflows/shared_get_modules.yml new file mode 100644 index 00000000..b36553f3 --- /dev/null +++ b/.github/workflows/shared_get_modules.yml @@ -0,0 +1,153 @@ +name: Shared Get Modules + +on: + workflow_call: + inputs: + environment: + description: environment reference i.e. 'prod' or 'dev' + required: true + type: string + infra_version: + description: "Version of infrastructure (terraform) to inspect" + required: true + type: string + ignore_task_modules: + description: "Whether to exclude task_* modules from emitted waves" + required: false + type: boolean + default: false + ignore_shared_artifact_modules: + description: "Whether to exclude shared artifact modules such as code_bucket and ecr from emitted waves" + required: false + type: boolean + default: false + ignore_oidc_module: + description: "Whether to exclude the oidc module from emitted waves" + required: false + type: boolean + default: false + outputs: + waves_json: + description: "Dependency-safe Terragrunt wave matrix JSON" + value: ${{ jobs.generate_waves.outputs.waves_json }} + wave_0_modules: + description: "Wave 0 module array" + value: ${{ jobs.generate_waves.outputs.wave_0_modules }} + wave_1_modules: + description: "Wave 1 module array" + value: ${{ jobs.generate_waves.outputs.wave_1_modules }} + wave_2_modules: + description: "Wave 2 module array" + value: ${{ jobs.generate_waves.outputs.wave_2_modules }} + wave_3_modules: + description: "Wave 3 module array" + value: ${{ jobs.generate_waves.outputs.wave_3_modules }} + +permissions: + id-token: write + contents: read + actions: read + +env: + AWS_OIDC_ROLE_ARN: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/${{ vars.PROJECT_NAME }}-${{ inputs.environment }}-github-oidc-role + AWS_REGION: ${{ vars.AWS_REGION }} + +jobs: + generate_waves: + runs-on: ubuntu-latest + outputs: + waves_json: ${{ steps.filtered_waves.outputs.waves_json }} + wave_0_modules: ${{ steps.wave_outputs.outputs.wave_0_modules }} + wave_1_modules: ${{ steps.wave_outputs.outputs.wave_1_modules }} + wave_2_modules: ${{ steps.wave_outputs.outputs.wave_2_modules }} + wave_3_modules: ${{ steps.wave_outputs.outputs.wave_3_modules }} + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ inputs.infra_version }} + + - uses: aws-actions/configure-aws-credentials@v6 + with: + role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }} + aws-region: ${{ env.AWS_REGION }} + + - name: Render Terragrunt graph + id: tg_graph + uses: ./.github/actions/terragrunt + with: + tg_directory: infra/live/${{ inputs.environment }}/aws + tg_action: graph + + - name: Convert graph output to compact JSON + id: tg_graph_json + uses: ./.github/actions/just + env: + TG_GRAPH_OUTPUT: ${{ steps.tg_graph.outputs.tg_graph_output }} + with: + justfile_path: justfile.ci + just_action: tg-graph-output-to-json ${{ inputs.environment }} + + - name: Build wave matrix JSON + id: waves_json + uses: ./.github/actions/just + env: + TG_GRAPH_JSON: ${{ steps.tg_graph_json.outputs.just_outputs }} + with: + justfile_path: justfile.ci + just_action: tg-graph-json-to-waves + + - name: Optionally exclude task modules from wave matrix + id: filtered_waves + shell: bash + env: + RAW_WAVES_JSON: ${{ steps.waves_json.outputs.just_outputs }} + IGNORE_TASK_MODULES: ${{ inputs.ignore_task_modules }} + IGNORE_SHARED_ARTIFACT_MODULES: ${{ inputs.ignore_shared_artifact_modules }} + IGNORE_OIDC_MODULE: ${{ inputs.ignore_oidc_module }} + run: | + echo "waves_json=$(jq -c ' + if $ignore_tasks then + map(.modules |= map(select(startswith("task_") | not))) + else + . + end + | if $ignore_shared_artifact_modules then + map(.modules |= map(select(. != "code_bucket" and . != "ecr"))) + else + . + end + | if $ignore_oidc_module then + map(.modules |= map(select(. != "oidc"))) + else + . + end + | map(select(.modules | length > 0)) + | to_entries + | map({wave: .key, modules: .value.modules}) + ' --argjson ignore_tasks "$IGNORE_TASK_MODULES" --argjson ignore_shared_artifact_modules "$IGNORE_SHARED_ARTIFACT_MODULES" --argjson ignore_oidc_module "$IGNORE_OIDC_MODULE" <<<"$RAW_WAVES_JSON")" >> "$GITHUB_OUTPUT" + + - name: Expose wave module arrays + id: wave_outputs + shell: bash + env: + WAVES_JSON: ${{ steps.filtered_waves.outputs.waves_json }} + run: | + echo "wave_0_modules=$(jq -c '.[0].modules // []' <<<"$WAVES_JSON")" >> "$GITHUB_OUTPUT" + echo "wave_1_modules=$(jq -c '.[1].modules // []' <<<"$WAVES_JSON")" >> "$GITHUB_OUTPUT" + echo "wave_2_modules=$(jq -c '.[2].modules // []' <<<"$WAVES_JSON")" >> "$GITHUB_OUTPUT" + echo "wave_3_modules=$(jq -c '.[3].modules // []' <<<"$WAVES_JSON")" >> "$GITHUB_OUTPUT" + + - name: Print Terragrunt wave matrix + shell: bash + env: + WAVES_JSON: ${{ steps.filtered_waves.outputs.waves_json }} + run: | + printf '%s\n' "$WAVES_JSON" | tee waves.json + { + echo "## Terragrunt Wave Matrix" + echo + echo '```json' + jq . waves.json + echo + echo '```' + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/shared_infra.yml b/.github/workflows/shared_infra.yml deleted file mode 100644 index aead7d34..00000000 --- a/.github/workflows/shared_infra.yml +++ /dev/null @@ -1,302 +0,0 @@ -name: Shared Infra - -on: - workflow_call: - inputs: - environment: - description: environment reference i.e. 'prod' or 'dev' - required: true - type: string - infra_version: - description: "Version of infrastructure (terraform) to be deployed" - required: true - type: string - code_bucket: - description: "Bucket containing build artifacts" - required: true - type: string - bootstrap_image_uri: - description: "Bootstrap ECS image URI" - required: false - type: string - default: "" - lambda_matrix: - required: false - type: string - default: "[]" - service_matrix: - required: false - type: string - default: "[]" - tg_action: - description: "Terragrunt action to run across the infra graph" - required: false - type: string - default: "apply" - plan_run_id: - description: "Optional unique run id used to derive saved plan artifact paths" - required: false - type: string - default: "" - - -concurrency: # only run one instance of workflow at any one time - group: infra-${{ inputs.environment }} - cancel-in-progress: false - -permissions: - id-token: write - contents: read - actions: read - -env: - AWS_OIDC_ROLE_ARN: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/${{ vars.PROJECT_NAME }}-${{ inputs.environment }}-github-oidc-role - AWS_REGION: ${{ vars.AWS_REGION }} - DOMAIN_NAME: ${{ vars.DOMAIN_NAME }} - TG_ENABLE_PLAN_ARTIFACTS: ${{ (inputs.tg_action == 'plan' || inputs.tg_action == 'apply_plan') && 'true' || 'false' }} - PLAN_ARTIFACT_RUN_ID: ${{ inputs.plan_run_id }} - TG_ACTION_LABEL: ${{ (inputs.tg_action == 'apply' || inputs.tg_action == 'apply_plan') && 'Apply' || inputs.tg_action == 'plan' && 'Plan' || inputs.tg_action == 'destroy' && 'Destroy' || inputs.tg_action == 'init' && 'Init' || 'Run' }} - -jobs: - oidc: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - with: - ref: ${{ inputs.infra_version }} - - - uses: aws-actions/configure-aws-credentials@v6 - with: - role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }} - aws-region: ${{ env.AWS_REGION }} - - - name: ${{ env.TG_ACTION_LABEL }} oidc role infra - uses: ./.github/actions/terragrunt - env: - TG_RESET_PLAN_ARTIFACT_BUCKET: "true" # this ensures that the plan artifact bucket is reset on every run, preventing stale artifacts from being used - with: - tg_directory: infra/live/${{ inputs.environment }}/aws/oidc - tg_action: ${{ inputs.tg_action }} - - messaging: - needs: oidc - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - with: - ref: ${{ inputs.infra_version }} - - - uses: aws-actions/configure-aws-credentials@v6 - with: - role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }} - aws-region: ${{ env.AWS_REGION }} - - - name: ${{ env.TG_ACTION_LABEL }} messaging infra - uses: ./.github/actions/terragrunt - with: - tg_directory: infra/live/${{ inputs.environment }}/aws/messaging - tg_action: ${{ inputs.tg_action }} - - observability: - needs: oidc - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - with: - ref: ${{ inputs.infra_version }} - - - uses: aws-actions/configure-aws-credentials@v6 - with: - role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }} - aws-region: ${{ env.AWS_REGION }} - - - name: ${{ env.TG_ACTION_LABEL }} observability infra - uses: ./.github/actions/terragrunt - with: - tg_directory: infra/live/${{ inputs.environment }}/aws/observability - tg_action: ${{ inputs.tg_action }} - - cognito: - needs: oidc - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - with: - ref: ${{ inputs.infra_version }} - - - uses: aws-actions/configure-aws-credentials@v6 - with: - role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }} - aws-region: ${{ env.AWS_REGION }} - - - name: ${{ env.TG_ACTION_LABEL }} cognito infra - uses: ./.github/actions/terragrunt - env: - TF_VAR_domain_name: ${{ env.DOMAIN_NAME }} - with: - tg_directory: infra/live/${{ inputs.environment }}/aws/cognito - tg_action: ${{ inputs.tg_action }} - - frontend: - needs: - - network - - cognito - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - with: - ref: ${{ inputs.infra_version }} - - - uses: aws-actions/configure-aws-credentials@v6 - with: - role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }} - aws-region: ${{ env.AWS_REGION }} - - - name: ${{ env.TG_ACTION_LABEL }} frontend infra - uses: ./.github/actions/terragrunt - env: - TF_VAR_domain_name: ${{ env.DOMAIN_NAME }} - with: - tg_directory: infra/live/${{ inputs.environment }}/aws/frontend - tg_action: ${{ inputs.tg_action }} - - cluster: - needs: oidc - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - with: - ref: ${{ inputs.infra_version }} - - - uses: aws-actions/configure-aws-credentials@v6 - with: - role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }} - aws-region: ${{ env.AWS_REGION }} - - - name: ${{ env.TG_ACTION_LABEL }} cluster infra - uses: ./.github/actions/terragrunt - with: - tg_directory: infra/live/${{ inputs.environment }}/aws/cluster - tg_action: ${{ inputs.tg_action }} - - security: - needs: oidc - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - with: - ref: ${{ inputs.infra_version }} - - - uses: aws-actions/configure-aws-credentials@v6 - with: - role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }} - aws-region: ${{ env.AWS_REGION }} - - - name: ${{ env.TG_ACTION_LABEL }} security infra - id: deploy-security - uses: ./.github/actions/terragrunt - with: - tg_directory: infra/live/${{ inputs.environment }}/aws/security - tg_action: ${{ inputs.tg_action }} - - database: - needs: - - oidc - - security - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - with: - ref: ${{ inputs.infra_version }} - - - uses: aws-actions/configure-aws-credentials@v6 - with: - role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }} - aws-region: ${{ env.AWS_REGION }} - - - name: ${{ env.TG_ACTION_LABEL }} database infra - uses: ./.github/actions/terragrunt - with: - tg_directory: infra/live/${{ inputs.environment }}/aws/database - tg_action: ${{ inputs.tg_action }} - - network: - needs: - - security - - cognito - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - with: - ref: ${{ inputs.infra_version }} - - - uses: aws-actions/configure-aws-credentials@v6 - with: - role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }} - aws-region: ${{ env.AWS_REGION }} - - - name: ${{ env.TG_ACTION_LABEL }} network infra - uses: ./.github/actions/terragrunt - with: - tg_directory: infra/live/${{ inputs.environment }}/aws/network - tg_action: ${{ inputs.tg_action }} - - lambdas: - needs: - - oidc - - security - - network - - database - - messaging - runs-on: ubuntu-latest - strategy: - fail-fast: false # this is to prevent terraform lock issues - matrix: - value: ${{ fromJson(inputs.lambda_matrix) }} - steps: - - uses: actions/checkout@v6 - with: - ref: ${{ inputs.infra_version }} - - - uses: aws-actions/configure-aws-credentials@v6 - with: - role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }} - aws-region: ${{ env.AWS_REGION }} - - - name: ${{ env.TG_ACTION_LABEL }} ${{ matrix.value }} infra - uses: ./.github/actions/terragrunt - with: - tg_directory: infra/live/${{ inputs.environment }}/aws/${{ matrix.value }} - tg_action: ${{ inputs.tg_action }} - - services: - needs: - - oidc - - security - - cluster - - network - - database - - messaging - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - value: ${{ fromJson(inputs.service_matrix) }} - steps: - - uses: actions/checkout@v6 - with: - ref: ${{ inputs.infra_version }} - - - uses: aws-actions/configure-aws-credentials@v6 - with: - role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }} - aws-region: ${{ env.AWS_REGION }} - - - name: ${{ env.TG_ACTION_LABEL }} ${{ matrix.value }} bootstrap service infra - uses: ./.github/actions/terragrunt - env: - TF_VAR_bootstrap: "true" - TF_VAR_bootstrap_image_uri: ${{ inputs.bootstrap_image_uri }} - with: - tg_directory: infra/live/${{ inputs.environment }}/aws/${{ matrix.value }} - tg_action: ${{ inputs.tg_action }} diff --git a/.github/workflows/shared_infra_apply.yml b/.github/workflows/shared_infra_apply.yml deleted file mode 100644 index 98ec6e1a..00000000 --- a/.github/workflows/shared_infra_apply.yml +++ /dev/null @@ -1,47 +0,0 @@ -name: Shared Infra Apply - -on: - workflow_call: - inputs: - environment: - description: environment reference i.e. 'prod' or 'dev' - required: true - type: string - infra_version: - description: "Version of infrastructure (terraform) to be applied" - required: true - type: string - code_bucket: - description: "Bucket containing build artifacts" - required: true - type: string - bootstrap_image_uri: - description: "Bootstrap ECS image URI" - required: false - type: string - default: "" - lambda_matrix: - required: false - type: string - default: "[]" - service_matrix: - required: false - type: string - default: "[]" - -permissions: - id-token: write - contents: read - actions: read - -jobs: - infra: - uses: ./.github/workflows/shared_infra.yml - with: - environment: ${{ inputs.environment }} - infra_version: ${{ inputs.infra_version }} - code_bucket: ${{ inputs.code_bucket }} - lambda_matrix: ${{ inputs.lambda_matrix }} - bootstrap_image_uri: ${{ inputs.bootstrap_image_uri }} - service_matrix: ${{ inputs.service_matrix }} - tg_action: apply diff --git a/.github/workflows/shared_infra_apply_from_plan.yml b/.github/workflows/shared_infra_apply_from_plan.yml index 8b1a8bdb..bf43ffe0 100644 --- a/.github/workflows/shared_infra_apply_from_plan.yml +++ b/.github/workflows/shared_infra_apply_from_plan.yml @@ -26,10 +26,12 @@ jobs: runs-on: ubuntu-latest outputs: infra_version: ${{ steps.read_metadata.outputs.infra_version }} - code_bucket: ${{ steps.read_metadata.outputs.code_bucket }} - lambda_matrix: ${{ steps.read_metadata.outputs.lambda_matrix }} bootstrap_image_uri: ${{ steps.read_metadata.outputs.bootstrap_image_uri }} - service_matrix: ${{ steps.read_metadata.outputs.service_matrix }} + waves_json: ${{ steps.read_metadata.outputs.waves_json }} + wave_0_modules: ${{ steps.read_metadata.outputs.wave_0_modules }} + wave_1_modules: ${{ steps.read_metadata.outputs.wave_1_modules }} + wave_2_modules: ${{ steps.read_metadata.outputs.wave_2_modules }} + wave_3_modules: ${{ steps.read_metadata.outputs.wave_3_modules }} steps: - uses: actions/checkout@v6 @@ -49,26 +51,158 @@ jobs: exit 1 fi + - name: Print recovered plan metadata JSON + shell: bash + run: | + echo "=== plan-metadata.json ===" + cat plan-metadata.json + - name: Read plan metadata id: read_metadata shell: bash run: | echo "infra_version=$(jq -r '.infra_version' plan-metadata.json)" >> "$GITHUB_OUTPUT" - echo "code_bucket=$(jq -r '.code_bucket' plan-metadata.json)" >> "$GITHUB_OUTPUT" - echo "lambda_matrix=$(jq -c '.lambda_matrix' plan-metadata.json)" >> "$GITHUB_OUTPUT" echo "bootstrap_image_uri=$(jq -r '.bootstrap_image_uri' plan-metadata.json)" >> "$GITHUB_OUTPUT" - echo "service_matrix=$(jq -c '.service_matrix' plan-metadata.json)" >> "$GITHUB_OUTPUT" + echo "waves_json=$(jq -c '.waves // []' plan-metadata.json)" >> "$GITHUB_OUTPUT" + echo "wave_0_modules=$(jq -c '.waves[0].modules // []' plan-metadata.json)" >> "$GITHUB_OUTPUT" + echo "wave_1_modules=$(jq -c '.waves[1].modules // []' plan-metadata.json)" >> "$GITHUB_OUTPUT" + echo "wave_2_modules=$(jq -c '.waves[2].modules // []' plan-metadata.json)" >> "$GITHUB_OUTPUT" + echo "wave_3_modules=$(jq -c '.waves[3].modules // []' plan-metadata.json)" >> "$GITHUB_OUTPUT" + + wave_0: + needs: metadata + name: 0 / ${{ matrix.module }} + if: ${{ needs.metadata.outputs.wave_0_modules != '[]' }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + module: ${{ fromJson(needs.metadata.outputs.wave_0_modules) }} + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ needs.metadata.outputs.infra_version }} + + - uses: aws-actions/configure-aws-credentials@v6 + with: + role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }} + aws-region: ${{ env.AWS_REGION }} + + - name: Download saved plan artifact + uses: actions/download-artifact@v7 + with: + name: terragrunt-plan-${{ inputs.environment }}-${{ matrix.module }} + github-token: ${{ github.token }} + run-id: ${{ inputs.plan_artifact_run_id }} + path: infra/live/${{ inputs.environment }}/aws/${{ matrix.module }} + + - name: Apply ${{ matrix.module }} infra from saved plan + uses: ./.github/actions/terragrunt + with: + tg_directory: infra/live/${{ inputs.environment }}/aws/${{ matrix.module }} + tg_action: apply_plan + + wave_1: + needs: + - metadata + - wave_0 + name: 1 / ${{ matrix.module }} + if: ${{ always() && needs.metadata.outputs.wave_1_modules != '[]' && (needs.wave_0.result == 'success' || needs.wave_0.result == 'skipped') }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + module: ${{ fromJson(needs.metadata.outputs.wave_1_modules) }} + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ needs.metadata.outputs.infra_version }} + + - uses: aws-actions/configure-aws-credentials@v6 + with: + role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }} + aws-region: ${{ env.AWS_REGION }} + + - name: Download saved plan artifact + uses: actions/download-artifact@v7 + with: + name: terragrunt-plan-${{ inputs.environment }}-${{ matrix.module }} + github-token: ${{ github.token }} + run-id: ${{ inputs.plan_artifact_run_id }} + path: infra/live/${{ inputs.environment }}/aws/${{ matrix.module }} - infra: + - name: Apply ${{ matrix.module }} infra from saved plan + uses: ./.github/actions/terragrunt + with: + tg_directory: infra/live/${{ inputs.environment }}/aws/${{ matrix.module }} + tg_action: apply_plan + + wave_2: needs: - metadata - uses: ./.github/workflows/shared_infra.yml - with: - environment: ${{ inputs.environment }} - infra_version: ${{ needs.metadata.outputs.infra_version }} - code_bucket: ${{ needs.metadata.outputs.code_bucket }} - lambda_matrix: ${{ needs.metadata.outputs.lambda_matrix }} - bootstrap_image_uri: ${{ needs.metadata.outputs.bootstrap_image_uri }} - service_matrix: ${{ needs.metadata.outputs.service_matrix }} - tg_action: apply_plan - plan_run_id: ${{ inputs.plan_artifact_run_id }} + - wave_1 + name: 2 / ${{ matrix.module }} + if: ${{ always() && needs.metadata.outputs.wave_2_modules != '[]' && (needs.wave_1.result == 'success' || needs.wave_1.result == 'skipped') }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + module: ${{ fromJson(needs.metadata.outputs.wave_2_modules) }} + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ needs.metadata.outputs.infra_version }} + + - uses: aws-actions/configure-aws-credentials@v6 + with: + role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }} + aws-region: ${{ env.AWS_REGION }} + + - name: Download saved plan artifact + uses: actions/download-artifact@v7 + with: + name: terragrunt-plan-${{ inputs.environment }}-${{ matrix.module }} + github-token: ${{ github.token }} + run-id: ${{ inputs.plan_artifact_run_id }} + path: infra/live/${{ inputs.environment }}/aws/${{ matrix.module }} + + - name: Apply ${{ matrix.module }} infra from saved plan + uses: ./.github/actions/terragrunt + with: + tg_directory: infra/live/${{ inputs.environment }}/aws/${{ matrix.module }} + tg_action: apply_plan + + wave_3: + needs: + - metadata + - wave_2 + name: 3 / ${{ matrix.module }} + if: ${{ always() && needs.metadata.outputs.wave_3_modules != '[]' && (needs.wave_2.result == 'success' || needs.wave_2.result == 'skipped') }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + module: ${{ fromJson(needs.metadata.outputs.wave_3_modules) }} + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ needs.metadata.outputs.infra_version }} + + - uses: aws-actions/configure-aws-credentials@v6 + with: + role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }} + aws-region: ${{ env.AWS_REGION }} + + - name: Download saved plan artifact + uses: actions/download-artifact@v7 + with: + name: terragrunt-plan-${{ inputs.environment }}-${{ matrix.module }} + github-token: ${{ github.token }} + run-id: ${{ inputs.plan_artifact_run_id }} + path: infra/live/${{ inputs.environment }}/aws/${{ matrix.module }} + + - name: Apply ${{ matrix.module }} infra from saved plan + uses: ./.github/actions/terragrunt + with: + tg_directory: infra/live/${{ inputs.environment }}/aws/${{ matrix.module }} + tg_action: apply_plan diff --git a/.github/workflows/shared_infra_apply_no_plan.yml b/.github/workflows/shared_infra_apply_no_plan.yml new file mode 100644 index 00000000..002bf3be --- /dev/null +++ b/.github/workflows/shared_infra_apply_no_plan.yml @@ -0,0 +1,142 @@ +name: Shared Infra Apply No Plan + +on: + workflow_call: + inputs: + environment: + description: environment reference i.e. 'prod' or 'dev' + required: true + type: string + infra_version: + description: "Version of infrastructure (terraform) to be applied" + required: true + type: string + bootstrap_image_uri: + description: "Bootstrap ECS image URI" + required: false + type: string + default: "" + +permissions: + id-token: write + contents: read + actions: read + +env: + AWS_OIDC_ROLE_ARN: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/${{ vars.PROJECT_NAME }}-${{ inputs.environment }}-github-oidc-role + AWS_REGION: ${{ vars.AWS_REGION }} + TG_ACTION_LABEL: "Apply" + +jobs: + generate_waves: + uses: ./.github/workflows/shared_get_modules.yml + with: + environment: ${{ inputs.environment }} + infra_version: ${{ inputs.infra_version }} + ignore_task_modules: true + + wave_0: + needs: generate_waves + name: 0 / ${{ matrix.module }} + if: ${{ needs.generate_waves.outputs.wave_0_modules != '[]' }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + module: ${{ fromJson(needs.generate_waves.outputs.wave_0_modules) }} + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ inputs.infra_version }} + + - uses: aws-actions/configure-aws-credentials@v6 + with: + role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }} + aws-region: ${{ env.AWS_REGION }} + + - name: ${{ env.TG_ACTION_LABEL }} ${{ matrix.module }} infra + uses: ./.github/actions/terragrunt + with: + tg_directory: infra/live/${{ inputs.environment }}/aws/${{ matrix.module }} + tg_action: apply + + wave_1: + needs: + - generate_waves + - wave_0 + name: 1 / ${{ matrix.module }} + if: ${{ always() && needs.generate_waves.outputs.wave_1_modules != '[]' && (needs.wave_0.result == 'success' || needs.wave_0.result == 'skipped') }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + module: ${{ fromJson(needs.generate_waves.outputs.wave_1_modules) }} + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ inputs.infra_version }} + + - uses: aws-actions/configure-aws-credentials@v6 + with: + role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }} + aws-region: ${{ env.AWS_REGION }} + + - name: ${{ env.TG_ACTION_LABEL }} ${{ matrix.module }} infra + uses: ./.github/actions/terragrunt + with: + tg_directory: infra/live/${{ inputs.environment }}/aws/${{ matrix.module }} + tg_action: apply + + wave_2: + needs: + - generate_waves + - wave_1 + name: 2 / ${{ matrix.module }} + if: ${{ always() && needs.generate_waves.outputs.wave_2_modules != '[]' && (needs.wave_1.result == 'success' || needs.wave_1.result == 'skipped') }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + module: ${{ fromJson(needs.generate_waves.outputs.wave_2_modules) }} + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ inputs.infra_version }} + + - uses: aws-actions/configure-aws-credentials@v6 + with: + role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }} + aws-region: ${{ env.AWS_REGION }} + + - name: ${{ env.TG_ACTION_LABEL }} ${{ matrix.module }} infra + uses: ./.github/actions/terragrunt + with: + tg_directory: infra/live/${{ inputs.environment }}/aws/${{ matrix.module }} + tg_action: apply + + wave_3: + needs: + - generate_waves + - wave_2 + name: 3 / ${{ matrix.module }} + if: ${{ always() && needs.generate_waves.outputs.wave_3_modules != '[]' && (needs.wave_2.result == 'success' || needs.wave_2.result == 'skipped') }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + module: ${{ fromJson(needs.generate_waves.outputs.wave_3_modules) }} + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ inputs.infra_version }} + + - uses: aws-actions/configure-aws-credentials@v6 + with: + role-to-assume: ${{ env.AWS_OIDC_ROLE_ARN }} + aws-region: ${{ env.AWS_REGION }} + + - name: ${{ env.TG_ACTION_LABEL }} ${{ matrix.module }} infra + uses: ./.github/actions/terragrunt + with: + tg_directory: infra/live/${{ inputs.environment }}/aws/${{ matrix.module }} + tg_action: apply diff --git a/.github/workflows/shared_infra_plan.yml b/.github/workflows/shared_infra_plan.yml index 1a31a7d0..8836cbdf 100644 --- a/.github/workflows/shared_infra_plan.yml +++ b/.github/workflows/shared_infra_plan.yml @@ -11,23 +11,11 @@ on: description: "Version of infrastructure (terraform) to be planned" required: true type: string - code_bucket: - description: "Bucket containing build artifacts" - required: true - type: string bootstrap_image_uri: description: "Bootstrap ECS image URI" required: false type: string default: "" - lambda_matrix: - required: false - type: string - default: "[]" - service_matrix: - required: false - type: string - default: "[]" outputs: plan_artifact_run_id: description: "Workflow run ID for the saved plan artifacts and metadata" @@ -40,9 +28,19 @@ permissions: env: AWS_OIDC_ROLE_ARN: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/${{ vars.PROJECT_NAME }}-${{ inputs.environment }}-github-oidc-role + AWS_REGION: ${{ vars.AWS_REGION }} + TG_ACTION_LABEL: "Plan" jobs: + generate_waves: + uses: ./.github/workflows/shared_get_modules.yml + with: + environment: ${{ inputs.environment }} + infra_version: ${{ inputs.infra_version }} + ignore_task_modules: true + metadata: + needs: generate_waves runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 @@ -55,10 +53,8 @@ jobs: cat > plan-metadata.json < graph.txt +``` + +That runs the same non-interactive Terragrunt graph command used in CI: + +```sh +cd infra/live/dev/aws +terragrunt run-all graph-dependencies \ + --terragrunt-non-interactive \ + --terragrunt-include-external-dependencies +``` + +To process that saved graph file into compact dependency JSON: + +```sh +just tg-graph-process graph.json dev +``` + +To return only changed saved-plan items as an object array, set the saved-plan env vars and run: + +```sh +BUCKET_NAME=700060376888-eu-west-2-aws-serverless-github-deploy-tfplan \ +TG_GRAPH_METADATA_PLAN_RUN_ID=26105102715 \ +just tg-graph-changed-items graph.json dev +``` + +To join the processed graph with saved-plan metadata for one plan run, set `TG_GRAPH_METADATA_PLAN_RUN_ID` and the plan bucket before running the processing command: + +```sh +BUCKET_NAME=700060376888-eu-west-2-aws-serverless-github-deploy-tfplan \ +TG_GRAPH_METADATA_PLAN_RUN_ID=26105102715 \ +just tg-graph-process graph.json dev +``` + ### Publish A Worker Message To publish directly to the shared worker SNS topic from your shell: @@ -235,13 +284,7 @@ The Cognito stack creates the user pool, app client, Hosted UI domain, and `read just cognito-create-readonly-user dev readonly@example.com 'ChangeMe123!' ``` -Set the GitHub environment variable `DOMAIN_NAME` to the hosted zone base domain, for example: - -```text -chrispsheehan.com -``` - -When that value is present, the frontend and Cognito stacks derive the deployed domain and auth callback/logout URLs automatically. Local Vite login still coexists through `http://localhost:5173`. +The frontend and Cognito stacks read `domain_name` from the shared Terragrunt global inputs in [infra/live/global_vars.hcl](infra/live/global_vars.hcl), which currently sets it to `chrispsheehan.com`. That keeps the deployed domain and auth callback/logout URLs consistent without extra CI wiring. Local Vite login still coexists through `http://localhost:5173`. ## Infra Deployment Use Cases @@ -268,7 +311,7 @@ Infrastructure apply and feature-code rollout are intentionally decoupled in thi - `*_infra` workflows apply infrastructure only - `*_code` workflows deploy feature code only - code deploy workflows publish the real Lambda versions and ECS task revisions into that pre-created deploy surface -- saved infra plans are stored in the shared S3 code bucket under `terragrunt_plan///...`, using the same artifact split as build outputs: `dev` writes to the `dev` code bucket and non-`dev` environments reuse the `ci` code bucket +- saved infra plans are stored as GitHub Actions artifacts keyed by workflow run id, with one run-level metadata artifact plus one per-stack plan artifact - Code artifact retention and infra-plan retention are configured separately in the shared code bucket module - rerunning infrastructure apply does not roll out new feature code - the shared Lambda and ECS module READMEs are the canonical source for bootstrap, rollout, and rollback details for each runtime shape diff --git a/REPO_INSTRUCTIONS.md b/REPO_INSTRUCTIONS.md index aa002a83..4979c723 100644 --- a/REPO_INSTRUCTIONS.md +++ b/REPO_INSTRUCTIONS.md @@ -40,6 +40,13 @@ These instructions apply to the entire repository. - runtime behavior: `lambdas/**/README.md` and `containers/**/README.md` - before editing, read the relevant local contract docs for the files you plan to touch and follow those contracts +## Script Ownership + +- reserve `infra/scripts/**` for Terraform or Terragrunt owned helper behavior that is part of the infra runtime contract +- Terragrunt graph rendering, saved-plan metadata lookups, and other helpers that shape the output of Terragrunt commands belong under `infra/scripts/**`, even if a `just` recipe or CI job invokes them +- prefer implementing GitHub Actions or workflow-only helper logic directly in `justfile.ci` when practical +- when a workflow-only helper needs more than a small recipe body, keep its ownership in the CI/workflow layer rather than under `infra/scripts/**` + ## Context Loading Order - load context lazily and only as needed @@ -54,6 +61,7 @@ These instructions apply to the entire repository. - when a request mentions external source code and asks how to build, make ready, or deploy it, interpret that as "understand the external app, then answer in terms of how this repository should implement or deploy it" unless the user explicitly redirects the work - read the relevant local contract docs before editing and follow them - prefer the smallest complete change that matches existing repo patterns +- remove stale code, temporary helpers, and abandoned experiment residue as part of the same change rather than leaving dead paths behind - verify related workflows, infra, docs, and downstream dependencies when the request affects shared behavior - state material assumptions when the intended shape is not fully explicit - when ambiguity is material or a wrong assumption could cause the repo shape or contract to drift, ask the user a clarifying question before editing @@ -97,13 +105,15 @@ These instructions apply to the entire repository. - verify runtime type (Lambda/ECS), deploy mode, and (for ECS) connection type and load-balancer shape - verify required infra resources exist (CodeDeploy app/deployment group, listeners/target groups, alarms, VPC link if applicable) +- before adding a Terragrunt `dependency` or `dependencies` path, verify the target live stack actually exists in that environment/repo slice - when changing reusable workflow contracts, compare every caller `with:` block to the callee `workflow_call.inputs` +- when a workflow input, output, or metadata field is no longer consumed, remove it from the shared contract and callers in the same change rather than leaving dead plumbing behind +- when changing Terragrunt `*.hcl` dependency edges, re-check the derived infra wave count; the current shared module-discovery/workflow contract only exposes `wave_0_modules`, `wave_1_modules`, and `wave_2_modules` - when adding or renaming Terraform module `output` values that are intended for Terragrunt `dependency..outputs` passthrough, verify every downstream consumer wrapper declares a `variable` with the exact same name - if that same-name output-to-variable contract does not hold yet, do not leave it implicit: either add the matching variables, or call out the mismatch explicitly before closing the task - check apply/deploy/destroy, and avoid unnecessary `terraform_remote_state` coupling (especially for fast-changing outputs) - for bootstrap-sensitive or plan-sensitive cross-stack contracts, prefer Terragrunt `dependency` inputs in the live stack and `mock_outputs` for non-mutating commands rather than reading upstream state directly inside Terraform modules - if CI plan failures are caused by missing upstream state, fix the contract shape first instead of papering over the issue with more direct `terraform_remote_state` reads -- when the same Terragrunt dependency wiring or mocks are needed across environments, centralize that shared config under `infra/live/dependencies/` in a capability-scoped helper such as `network.hcl` and have environment stacks read it rather than duplicating the same blocks in `dev`, `prod`, or `ci` - keep this approach visible to users as well: when you introduce or expand this pattern, update the top-level `README.md` so the bootstrap-friendly mock strategy is documented outside agent-only instructions - if you intentionally add a Terraform `data "terraform_remote_state"` block, add a `# remote_state_reason: ...` comment immediately above it explaining why Terragrunt `dependency` plus `mock_outputs` is not practical for that case - if you intentionally add a Terraform `data "terraform_remote_state"` block, add a `# remote_state_reason: ...` comment immediately above it explaining why Terragrunt `dependency` plus `mock_outputs` is not practical for that case diff --git a/infra/README.md b/infra/README.md index 5c9264c0..e90e07ed 100644 --- a/infra/README.md +++ b/infra/README.md @@ -34,10 +34,7 @@ Shared artifact names also follow naming conventions from `infra/root.hcl`: - dedicated saved-plan bucket: `---tfplan` - code bucket: `-code` - ECS ECR repository: `-ecr` -- saved Terragrunt plan artifacts: `s3:///terragrunt_plan///...` -- plan-bucket retention: `infra_plan_artifact_expiration_days` applies an S3 lifecycle rule to `terragrunt_plan/` in the dedicated saved-plan bucket -- during `terragrunt init` and saved-plan `plan`, the root hook ensures the dedicated saved-plan bucket exists; interactive runs prompt before creation and non-interactive runs fail if no prompt is possible -- to reapply the configured `infra_plan_artifact_expiration_days` lifecycle rule locally for an existing bucket, rerun with `TG_RESET_PLAN_ARTIFACT_BUCKET=true` +- saved Terragrunt plan artifacts are stored as GitHub Actions artifacts keyed by workflow run id So a stack at: @@ -71,7 +68,7 @@ stores state at: - `cognito` Owns the Cognito user pool, frontend app client, Hosted UI domain, and read-only user group. - `frontend` - Owns the derived CloudFront custom domain, ACM certificate in `us-east-1`, and Route53 alias records using the required `DOMAIN_NAME` workflow env input. + Owns the derived CloudFront custom domain, ACM certificate in `us-east-1`, and Route53 alias records using the Terragrunt `domain_name` input from `infra/live/global_vars.hcl`. - `observability` Owns the shared CloudWatch dashboard used to inspect Lambda and ECS logs in one console place. - `database` @@ -156,7 +153,8 @@ That `containers/lib` directory is helper code only and is not treated as a depl - build workflows produce Lambda zips and container images - `*_infra` wrappers need the inputs required to apply infra safely, such as directory-derived stack matrices and any artifact-derived bootstrap references - in `prod`, the `*_infra` wrappers read shared artifact resources from `ci` but only apply service and task stacks in `prod` -- saved `plan` / `apply_plan` artifacts live in the dedicated plan bucket under `terragrunt_plan///...` +- saved `plan` / `apply_plan` artifacts live in GitHub Actions artifacts keyed by workflow run id +- each saved-plan stack always uploads `terragrunt.plan.meta.json`; the binary `terragrunt.tfplan` and rendered `terragrunt.plan.txt` are uploaded only when the plan contains real changes - deploy workflows: - publish Lambda versions and use Lambda CodeDeploy - optionally invoke the `migrations` Lambda when it is part of the Lambda deploy matrix @@ -198,23 +196,17 @@ just --justfile justfile.deploy lambda-get-version just --justfile justfile.deploy frontend-build ``` -For a local saved-plan run that can upload plan artifacts through the normal repo wrapper, enable artifact mode, provide a unique run id, and pass the Terragrunt operation as one quoted argument: +For a local saved-plan run, pass the Terragrunt operation as one quoted argument: ```sh -TG_ENABLE_PLAN_ARTIFACTS=true \ -PLAN_ARTIFACT_RUN_ID="local-example-run" \ just tg dev aws/oidc 'plan -out=terragrunt.tfplan' ``` -The `tg` recipe treats the final argument as the Terragrunt operation string, so quoting lets you pass flags such as `-out=...` through the wrapper. The current saved-plan hook expects the binary plan filename to be `terragrunt.tfplan`; if you choose a different `-out` filename, the upload hook will not find it. - -Per-stack saved-plan bundles in S3 use the live stack identity rather than your full local filesystem path, for example `terragrunt-plan-dev-aws-oidc`. +The `tg` recipe treats the final argument as the Terragrunt operation string, so quoting lets you pass flags such as `-out=...` through the wrapper. The workflow saved-plan path expects the binary plan filename to be `terragrunt.tfplan`. To apply that same saved plan later, reuse the same run id: ```sh -TG_ENABLE_PLAN_ARTIFACTS=true \ -PLAN_ARTIFACT_RUN_ID="local-example-run" \ just tg dev aws/oidc 'apply terragrunt.tfplan' ``` diff --git a/infra/live/ci/aws/code_bucket/terragrunt.hcl b/infra/live/ci/aws/code_bucket/terragrunt.hcl index ed01f2ed..1d6a045b 100644 --- a/infra/live/ci/aws/code_bucket/terragrunt.hcl +++ b/infra/live/ci/aws/code_bucket/terragrunt.hcl @@ -2,6 +2,10 @@ include "root" { path = find_in_parent_folders("root.hcl") } +dependencies { + paths = ["../oidc"] +} + terraform { source = "../../../../modules//aws//_shared//code_bucket" } diff --git a/infra/live/ci/aws/ecr/terragrunt.hcl b/infra/live/ci/aws/ecr/terragrunt.hcl index 1d671831..881bf7b4 100644 --- a/infra/live/ci/aws/ecr/terragrunt.hcl +++ b/infra/live/ci/aws/ecr/terragrunt.hcl @@ -2,6 +2,10 @@ include "root" { path = find_in_parent_folders("root.hcl") } +dependencies { + paths = ["../oidc"] +} + terraform { source = "../../../../modules//aws//_shared//ecr" } diff --git a/infra/live/dependencies/cluster.hcl b/infra/live/dependencies/cluster.hcl deleted file mode 100644 index 2bbd9e09..00000000 --- a/infra/live/dependencies/cluster.hcl +++ /dev/null @@ -1,15 +0,0 @@ -dependency "cluster" { - config_path = "${get_original_terragrunt_dir()}/../cluster" - - mock_outputs = { - cluster_id = "mock-cluster-id" - cluster_name = "mock-cluster" - } - - mock_outputs_allowed_terraform_commands = ["validate", "plan", "destroy", "init", "show"] -} - -inputs = { - cluster_id = dependency.cluster.outputs.cluster_id - cluster_name = dependency.cluster.outputs.cluster_name -} diff --git a/infra/live/dependencies/database.hcl b/infra/live/dependencies/database.hcl deleted file mode 100644 index 02dd17c6..00000000 --- a/infra/live/dependencies/database.hcl +++ /dev/null @@ -1,21 +0,0 @@ -dependency "database" { - config_path = "${get_original_terragrunt_dir()}/../database" - - mock_outputs = { - database_credentials_secret_arn = "arn:aws:secretsmanager:eu-west-2:111111111111:secret:mock-database-credentials" - database_readwrite_endpoint = "mock-database.cluster-abcdefghijkl.eu-west-2.rds.amazonaws.com" - database_name = "app" - database_port = 5432 - database_cluster_identifier = "mock-database-cluster" - } - - mock_outputs_allowed_terraform_commands = ["validate", "plan", "destroy", "init", "show"] -} - -inputs = { - database_credentials_secret_arn = dependency.database.outputs.database_credentials_secret_arn - database_readwrite_endpoint = dependency.database.outputs.database_readwrite_endpoint - database_name = dependency.database.outputs.database_name - database_port = dependency.database.outputs.database_port - database_cluster_identifier = dependency.database.outputs.database_cluster_identifier -} diff --git a/infra/live/dependencies/frontend.hcl b/infra/live/dependencies/frontend.hcl deleted file mode 100644 index 01dba858..00000000 --- a/infra/live/dependencies/frontend.hcl +++ /dev/null @@ -1,30 +0,0 @@ -dependency "network" { - config_path = "${get_original_terragrunt_dir()}/../network" - - mock_outputs = { - api_invoke_url = "https://mockapi123.execute-api.eu-west-2.amazonaws.com" - } - - mock_outputs_allowed_terraform_commands = ["validate", "plan", "destroy", "init", "show"] -} - -dependency "cognito" { - config_path = "${get_original_terragrunt_dir()}/../cognito" - - mock_outputs = { - auth_user_pool_id = "eu-west-2_mock" - auth_user_pool_client_id = "mock-user-pool-client-id" - auth_hosted_ui_url = "https://mock-domain.auth.eu-west-2.amazoncognito.com" - auth_readonly_group_name = "readonly" - } - - mock_outputs_allowed_terraform_commands = ["validate", "plan", "destroy", "init", "show"] -} - -inputs = { - api_invoke_url = dependency.network.outputs.api_invoke_url - auth_user_pool_id = dependency.cognito.outputs.auth_user_pool_id - auth_user_pool_client_id = dependency.cognito.outputs.auth_user_pool_client_id - auth_hosted_ui_url = dependency.cognito.outputs.auth_hosted_ui_url - auth_readonly_group_name = dependency.cognito.outputs.auth_readonly_group_name -} diff --git a/infra/live/dependencies/messaging.hcl b/infra/live/dependencies/messaging.hcl deleted file mode 100644 index 12399468..00000000 --- a/infra/live/dependencies/messaging.hcl +++ /dev/null @@ -1,22 +0,0 @@ -dependency "messaging" { - config_path = "${get_original_terragrunt_dir()}/../messaging" - - mock_outputs = { - worker_topic_name = "mock-worker-events" - worker_topic_arn = "arn:aws:sns:eu-west-2:111111111111:mock-worker-events" - worker_topic_publish_policy_arn = "arn:aws:iam::111111111111:policy/mock-worker-topic-publish" - lambda_worker_queue_name = "mock-lambda-worker-queue" - lambda_worker_queue_arn = "arn:aws:sqs:eu-west-2:111111111111:mock-lambda-worker-queue" - lambda_worker_queue_url = "https://sqs.eu-west-2.amazonaws.com/111111111111/mock-lambda-worker-queue" - lambda_worker_queue_read_policy_arn = "arn:aws:iam::111111111111:policy/mock-lambda-worker-queue-read" - lambda_worker_dead_letter_queue_name = "mock-lambda-worker-dlq" - lambda_worker_dead_letter_queue_url = "https://sqs.eu-west-2.amazonaws.com/111111111111/mock-lambda-worker-dlq" - ecs_worker_queue_name = "mock-ecs-worker-queue" - ecs_worker_queue_url = "https://sqs.eu-west-2.amazonaws.com/111111111111/mock-ecs-worker-queue" - ecs_worker_queue_read_policy_arn = "arn:aws:iam::111111111111:policy/mock-ecs-worker-queue-read" - } - - mock_outputs_allowed_terraform_commands = ["validate", "plan", "destroy", "init", "show"] -} - -inputs = dependency.messaging.outputs diff --git a/infra/live/dependencies/network.hcl b/infra/live/dependencies/network.hcl deleted file mode 100644 index de2a1de5..00000000 --- a/infra/live/dependencies/network.hcl +++ /dev/null @@ -1,35 +0,0 @@ -dependency "network" { - config_path = "${get_original_terragrunt_dir()}/../network" - - mock_outputs = { - default_target_group_arn = "arn:aws:elasticloadbalancing:eu-west-2:111111111111:targetgroup/mock-default/1234567890abcdef" - load_balancer_arn = "arn:aws:elasticloadbalancing:eu-west-2:111111111111:loadbalancer/app/mock-internal/1234567890abcdef" - default_http_listener_arn = "arn:aws:elasticloadbalancing:eu-west-2:111111111111:listener/app/mock-internal/1234567890abcdef/abcdef1234567890" - load_balancer_arn_suffix = "app/mock-internal/1234567890abcdef" - target_group_arn_suffix = "targetgroup/mock-default/1234567890abcdef" - internal_invoke_url = "http://mock-internal-123456.eu-west-2.elb.amazonaws.com" - api_id = "mockapi123" - api_invoke_url = "https://mockapi123.execute-api.eu-west-2.amazonaws.com" - api_execution_arn = "arn:aws:execute-api:eu-west-2:111111111111:mockapi123" - api_stage_name = "$default" - vpc_link_id = "vpclink-mock123" - http_api_authorizer_id = "auth-mock123" - } - - mock_outputs_allowed_terraform_commands = ["validate", "plan", "destroy", "init", "show"] -} - -inputs = { - network_default_target_group_arn = dependency.network.outputs.default_target_group_arn - network_load_balancer_arn = dependency.network.outputs.load_balancer_arn - network_default_http_listener_arn = dependency.network.outputs.default_http_listener_arn - network_load_balancer_arn_suffix = dependency.network.outputs.load_balancer_arn_suffix - network_target_group_arn_suffix = dependency.network.outputs.target_group_arn_suffix - network_internal_invoke_url = dependency.network.outputs.internal_invoke_url - network_api_id = dependency.network.outputs.api_id - network_api_invoke_url = dependency.network.outputs.api_invoke_url - network_api_execution_arn = dependency.network.outputs.api_execution_arn - network_api_stage_name = dependency.network.outputs.api_stage_name - network_vpc_link_id = dependency.network.outputs.vpc_link_id - network_http_api_authorizer_id = dependency.network.outputs.http_api_authorizer_id -} diff --git a/infra/live/dependencies/security.hcl b/infra/live/dependencies/security.hcl deleted file mode 100644 index 07ac0a9e..00000000 --- a/infra/live/dependencies/security.hcl +++ /dev/null @@ -1,14 +0,0 @@ -dependency "security" { - config_path = "${get_original_terragrunt_dir()}/../security" - - mock_outputs = { - load_balancer_sg = "sg-00000000000000001" - api_vpc_link_sg = "sg-00000000000000002" - vpc_endpoint_sg = "sg-00000000000000003" - ecs_sg = "sg-00000000000000004" - runtime_sg = "sg-00000000000000005" - postgres_sg = "sg-00000000000000006" - } - - mock_outputs_allowed_terraform_commands = ["validate", "plan", "destroy", "init", "show"] -} diff --git a/infra/live/dev/aws/cluster/terragrunt.hcl b/infra/live/dev/aws/cluster/terragrunt.hcl index 8d29bd5d..b52fb59a 100644 --- a/infra/live/dev/aws/cluster/terragrunt.hcl +++ b/infra/live/dev/aws/cluster/terragrunt.hcl @@ -2,6 +2,10 @@ include "root" { path = find_in_parent_folders("root.hcl") } +dependencies { + paths = ["../oidc"] +} + terraform { source = "../../../../modules//aws//_shared//cluster" } diff --git a/infra/live/dev/aws/code_bucket/terragrunt.hcl b/infra/live/dev/aws/code_bucket/terragrunt.hcl index ed01f2ed..1d6a045b 100644 --- a/infra/live/dev/aws/code_bucket/terragrunt.hcl +++ b/infra/live/dev/aws/code_bucket/terragrunt.hcl @@ -2,6 +2,10 @@ include "root" { path = find_in_parent_folders("root.hcl") } +dependencies { + paths = ["../oidc"] +} + terraform { source = "../../../../modules//aws//_shared//code_bucket" } diff --git a/infra/live/dev/aws/cognito/terragrunt.hcl b/infra/live/dev/aws/cognito/terragrunt.hcl index 6e2da3c4..5f937331 100644 --- a/infra/live/dev/aws/cognito/terragrunt.hcl +++ b/infra/live/dev/aws/cognito/terragrunt.hcl @@ -2,6 +2,10 @@ include "root" { path = find_in_parent_folders("root.hcl") } +dependencies { + paths = ["../oidc"] +} + terraform { source = "../../../../modules//aws//cognito" } diff --git a/infra/live/dev/aws/database/terragrunt.hcl b/infra/live/dev/aws/database/terragrunt.hcl index c6ee9ddc..428ee1c2 100644 --- a/infra/live/dev/aws/database/terragrunt.hcl +++ b/infra/live/dev/aws/database/terragrunt.hcl @@ -2,8 +2,20 @@ include "root" { path = find_in_parent_folders("root.hcl") } -include "security" { - path = find_in_parent_folders("dependencies/security.hcl") +dependency "security" { + config_path = "${get_original_terragrunt_dir()}/../security" + + mock_outputs = { + load_balancer_sg = "sg-00000000000000001" + api_vpc_link_sg = "sg-00000000000000002" + vpc_endpoint_sg = "sg-00000000000000003" + ecs_sg = "sg-00000000000000004" + runtime_sg = "sg-00000000000000005" + postgres_sg = "sg-00000000000000006" + } + + mock_outputs_merge_strategy_with_state = "shallow" + mock_outputs_allowed_terraform_commands = ["validate", "plan", "destroy", "init", "show", "graph-dependencies", "output-module-groups"] } terraform { diff --git a/infra/live/dev/aws/ecr/terragrunt.hcl b/infra/live/dev/aws/ecr/terragrunt.hcl index 1d671831..881bf7b4 100644 --- a/infra/live/dev/aws/ecr/terragrunt.hcl +++ b/infra/live/dev/aws/ecr/terragrunt.hcl @@ -2,6 +2,10 @@ include "root" { path = find_in_parent_folders("root.hcl") } +dependencies { + paths = ["../oidc"] +} + terraform { source = "../../../../modules//aws//_shared//ecr" } diff --git a/infra/live/dev/aws/frontend/terragrunt.hcl b/infra/live/dev/aws/frontend/terragrunt.hcl index fa22a838..bc2220f9 100644 --- a/infra/live/dev/aws/frontend/terragrunt.hcl +++ b/infra/live/dev/aws/frontend/terragrunt.hcl @@ -2,12 +2,39 @@ include "root" { path = find_in_parent_folders("root.hcl") } -locals { - frontend = read_terragrunt_config(find_in_parent_folders("dependencies/frontend.hcl")) +dependency "network" { + config_path = "${get_original_terragrunt_dir()}/../network" + + mock_outputs = { + api_invoke_url = "https://mockapi123.execute-api.eu-west-2.amazonaws.com" + } + + mock_outputs_merge_strategy_with_state = "shallow" + mock_outputs_allowed_terraform_commands = ["validate", "plan", "destroy", "init", "show", "graph-dependencies", "output-module-groups"] +} + +dependency "cognito" { + config_path = "${get_original_terragrunt_dir()}/../cognito" + + mock_outputs = { + auth_user_pool_id = "eu-west-2_mock" + auth_user_pool_client_id = "mock-user-pool-client-id" + auth_hosted_ui_url = "https://mock-domain.auth.eu-west-2.amazoncognito.com" + auth_readonly_group_name = "readonly" + } + + mock_outputs_merge_strategy_with_state = "shallow" + mock_outputs_allowed_terraform_commands = ["validate", "plan", "destroy", "init", "show", "graph-dependencies", "output-module-groups"] } terraform { source = "../../../../modules//aws//frontend" } -inputs = local.frontend.inputs +inputs = { + api_invoke_url = dependency.network.outputs.api_invoke_url + auth_user_pool_id = dependency.cognito.outputs.auth_user_pool_id + auth_user_pool_client_id = dependency.cognito.outputs.auth_user_pool_client_id + auth_hosted_ui_url = dependency.cognito.outputs.auth_hosted_ui_url + auth_readonly_group_name = dependency.cognito.outputs.auth_readonly_group_name +} diff --git a/infra/live/dev/aws/lambda_api/terragrunt.hcl b/infra/live/dev/aws/lambda_api/terragrunt.hcl index 957503d9..17e67ce6 100644 --- a/infra/live/dev/aws/lambda_api/terragrunt.hcl +++ b/infra/live/dev/aws/lambda_api/terragrunt.hcl @@ -2,9 +2,63 @@ include "root" { path = find_in_parent_folders("root.hcl") } -locals { - network = read_terragrunt_config(find_in_parent_folders("dependencies/network.hcl")) - messaging = read_terragrunt_config(find_in_parent_folders("dependencies/messaging.hcl")) +dependencies { + paths = ["../security", "../database"] +} + +dependency "network" { + config_path = "${get_original_terragrunt_dir()}/../network" + + mock_outputs = { + default_target_group_arn = "arn:aws:elasticloadbalancing:eu-west-2:111111111111:targetgroup/mock-default/1234567890abcdef" + load_balancer_arn = "arn:aws:elasticloadbalancing:eu-west-2:111111111111:loadbalancer/app/mock-internal/1234567890abcdef" + default_http_listener_arn = "arn:aws:elasticloadbalancing:eu-west-2:111111111111:listener/app/mock-internal/1234567890abcdef/abcdef1234567890" + load_balancer_arn_suffix = "app/mock-internal/1234567890abcdef" + target_group_arn_suffix = "targetgroup/mock-default/1234567890abcdef" + internal_invoke_url = "http://mock-internal-123456.eu-west-2.elb.amazonaws.com" + api_id = "mockapi123" + api_invoke_url = "https://mockapi123.execute-api.eu-west-2.amazonaws.com" + api_execution_arn = "arn:aws:execute-api:eu-west-2:111111111111:mockapi123" + api_stage_name = "$default" + vpc_link_id = "vpclink-mock123" + http_api_authorizer_id = "auth-mock123" + } + + mock_outputs_merge_strategy_with_state = "shallow" + mock_outputs_allowed_terraform_commands = ["validate", "plan", "destroy", "init", "show", "graph-dependencies", "output-module-groups"] +} + +dependency "messaging" { + config_path = "${get_original_terragrunt_dir()}/../messaging" + + mock_outputs = { + worker_topic_name = "mock-worker-events" + worker_topic_arn = "arn:aws:sns:eu-west-2:111111111111:mock-worker-events" + worker_topic_publish_policy_arn = "arn:aws:iam::111111111111:policy/mock-worker-topic-publish" + lambda_worker_queue_name = "mock-lambda-worker-queue" + lambda_worker_queue_arn = "arn:aws:sqs:eu-west-2:111111111111:mock-lambda-worker-queue" + lambda_worker_queue_url = "https://sqs.eu-west-2.amazonaws.com/111111111111/mock-lambda-worker-queue" + lambda_worker_queue_read_policy_arn = "arn:aws:iam::111111111111:policy/mock-lambda-worker-queue-read" + lambda_worker_dead_letter_queue_name = "mock-lambda-worker-dlq" + lambda_worker_dead_letter_queue_url = "https://sqs.eu-west-2.amazonaws.com/111111111111/mock-lambda-worker-dlq" + ecs_worker_queue_name = "mock-ecs-worker-queue" + ecs_worker_queue_url = "https://sqs.eu-west-2.amazonaws.com/111111111111/mock-ecs-worker-queue" + ecs_worker_queue_read_policy_arn = "arn:aws:iam::111111111111:policy/mock-ecs-worker-queue-read" + } + + mock_outputs_merge_strategy_with_state = "shallow" + mock_outputs_allowed_terraform_commands = ["validate", "plan", "destroy", "init", "show", "graph-dependencies", "output-module-groups"] +} + +dependency "code_bucket" { + config_path = "${get_original_terragrunt_dir()}/../code_bucket" + + mock_outputs = { + bucket = "mock-code-bucket" + } + + mock_outputs_merge_strategy_with_state = "shallow" + mock_outputs_allowed_terraform_commands = ["validate", "plan", "destroy", "init", "show", "graph-dependencies", "output-module-groups"] } terraform { @@ -12,8 +66,24 @@ terraform { } inputs = merge( - local.network.inputs, - local.messaging.inputs, + { + code_bucket = dependency.code_bucket.outputs.bucket + }, + { + network_default_target_group_arn = dependency.network.outputs.default_target_group_arn + network_load_balancer_arn = dependency.network.outputs.load_balancer_arn + network_default_http_listener_arn = dependency.network.outputs.default_http_listener_arn + network_load_balancer_arn_suffix = dependency.network.outputs.load_balancer_arn_suffix + network_target_group_arn_suffix = dependency.network.outputs.target_group_arn_suffix + network_internal_invoke_url = dependency.network.outputs.internal_invoke_url + network_api_id = dependency.network.outputs.api_id + network_api_invoke_url = dependency.network.outputs.api_invoke_url + network_api_execution_arn = dependency.network.outputs.api_execution_arn + network_api_stage_name = dependency.network.outputs.api_stage_name + network_vpc_link_id = dependency.network.outputs.vpc_link_id + network_http_api_authorizer_id = dependency.network.outputs.http_api_authorizer_id + }, + dependency.messaging.outputs, { api_5xx_alarm_threshold = 20.0 api_5xx_alarm_evaluation_periods = 1 diff --git a/infra/live/dev/aws/lambda_worker/terragrunt.hcl b/infra/live/dev/aws/lambda_worker/terragrunt.hcl index bdb94fdd..ccd15214 100644 --- a/infra/live/dev/aws/lambda_worker/terragrunt.hcl +++ b/infra/live/dev/aws/lambda_worker/terragrunt.hcl @@ -2,8 +2,41 @@ include "root" { path = find_in_parent_folders("root.hcl") } -locals { - messaging = read_terragrunt_config(find_in_parent_folders("dependencies/messaging.hcl")) +dependencies { + paths = ["../security", "../network", "../database"] +} + +dependency "messaging" { + config_path = "${get_original_terragrunt_dir()}/../messaging" + + mock_outputs = { + worker_topic_name = "mock-worker-events" + worker_topic_arn = "arn:aws:sns:eu-west-2:111111111111:mock-worker-events" + worker_topic_publish_policy_arn = "arn:aws:iam::111111111111:policy/mock-worker-topic-publish" + lambda_worker_queue_name = "mock-lambda-worker-queue" + lambda_worker_queue_arn = "arn:aws:sqs:eu-west-2:111111111111:mock-lambda-worker-queue" + lambda_worker_queue_url = "https://sqs.eu-west-2.amazonaws.com/111111111111/mock-lambda-worker-queue" + lambda_worker_queue_read_policy_arn = "arn:aws:iam::111111111111:policy/mock-lambda-worker-queue-read" + lambda_worker_dead_letter_queue_name = "mock-lambda-worker-dlq" + lambda_worker_dead_letter_queue_url = "https://sqs.eu-west-2.amazonaws.com/111111111111/mock-lambda-worker-dlq" + ecs_worker_queue_name = "mock-ecs-worker-queue" + ecs_worker_queue_url = "https://sqs.eu-west-2.amazonaws.com/111111111111/mock-ecs-worker-queue" + ecs_worker_queue_read_policy_arn = "arn:aws:iam::111111111111:policy/mock-ecs-worker-queue-read" + } + + mock_outputs_merge_strategy_with_state = "shallow" + mock_outputs_allowed_terraform_commands = ["validate", "plan", "destroy", "init", "show", "graph-dependencies", "output-module-groups"] +} + +dependency "code_bucket" { + config_path = "${get_original_terragrunt_dir()}/../code_bucket" + + mock_outputs = { + bucket = "mock-code-bucket" + } + + mock_outputs_merge_strategy_with_state = "shallow" + mock_outputs_allowed_terraform_commands = ["validate", "plan", "destroy", "init", "show", "graph-dependencies", "output-module-groups"] } terraform { @@ -11,7 +44,10 @@ terraform { } inputs = merge( - local.messaging.inputs, + { + code_bucket = dependency.code_bucket.outputs.bucket + }, + dependency.messaging.outputs, { sqs_dlq_alarm_threshold = 1 # fail when any messages are in the DLQ (quick fail for testing) sqs_dlq_alarm_evaluation_periods = 1 diff --git a/infra/live/dev/aws/messaging/terragrunt.hcl b/infra/live/dev/aws/messaging/terragrunt.hcl index b85cf717..8cf46acc 100644 --- a/infra/live/dev/aws/messaging/terragrunt.hcl +++ b/infra/live/dev/aws/messaging/terragrunt.hcl @@ -2,6 +2,10 @@ include "root" { path = find_in_parent_folders("root.hcl") } +dependencies { + paths = ["../oidc"] +} + terraform { source = "../../../../modules//aws//messaging" } diff --git a/infra/live/dev/aws/migrations/terragrunt.hcl b/infra/live/dev/aws/migrations/terragrunt.hcl index edc1b7f2..0633a506 100644 --- a/infra/live/dev/aws/migrations/terragrunt.hcl +++ b/infra/live/dev/aws/migrations/terragrunt.hcl @@ -2,12 +2,46 @@ include "root" { path = find_in_parent_folders("root.hcl") } -include "security" { - path = find_in_parent_folders("dependencies/security.hcl") +dependency "security" { + config_path = "${get_original_terragrunt_dir()}/../security" + + mock_outputs = { + load_balancer_sg = "sg-00000000000000001" + api_vpc_link_sg = "sg-00000000000000002" + vpc_endpoint_sg = "sg-00000000000000003" + ecs_sg = "sg-00000000000000004" + runtime_sg = "sg-00000000000000005" + postgres_sg = "sg-00000000000000006" + } + + mock_outputs_merge_strategy_with_state = "shallow" + mock_outputs_allowed_terraform_commands = ["validate", "plan", "destroy", "init", "show", "graph-dependencies", "output-module-groups"] +} + +dependency "database" { + config_path = "${get_original_terragrunt_dir()}/../database" + + mock_outputs = { + database_credentials_secret_arn = "arn:aws:secretsmanager:eu-west-2:111111111111:secret:mock-database-credentials" + database_readwrite_endpoint = "mock-database.cluster-abcdefghijkl.eu-west-2.rds.amazonaws.com" + database_name = "app" + database_port = 5432 + database_cluster_identifier = "mock-database-cluster" + } + + mock_outputs_merge_strategy_with_state = "shallow" + mock_outputs_allowed_terraform_commands = ["validate", "plan", "destroy", "init", "show", "graph-dependencies", "output-module-groups"] } -locals { - database = read_terragrunt_config(find_in_parent_folders("dependencies/database.hcl")) +dependency "code_bucket" { + config_path = "${get_original_terragrunt_dir()}/../code_bucket" + + mock_outputs = { + bucket = "mock-code-bucket" + } + + mock_outputs_merge_strategy_with_state = "shallow" + mock_outputs_allowed_terraform_commands = ["validate", "plan", "destroy", "init", "show", "graph-dependencies", "output-module-groups"] } terraform { @@ -15,8 +49,17 @@ terraform { } inputs = merge( + { + code_bucket = dependency.code_bucket.outputs.bucket + }, { runtime_security_group_id = dependency.security.outputs.runtime_sg }, - local.database.inputs, + { + database_credentials_secret_arn = dependency.database.outputs.database_credentials_secret_arn + database_readwrite_endpoint = dependency.database.outputs.database_readwrite_endpoint + database_name = dependency.database.outputs.database_name + database_port = dependency.database.outputs.database_port + database_cluster_identifier = dependency.database.outputs.database_cluster_identifier + }, ) diff --git a/infra/live/dev/aws/network/terragrunt.hcl b/infra/live/dev/aws/network/terragrunt.hcl index 93443bf3..0824c6dc 100644 --- a/infra/live/dev/aws/network/terragrunt.hcl +++ b/infra/live/dev/aws/network/terragrunt.hcl @@ -2,8 +2,20 @@ include "root" { path = find_in_parent_folders("root.hcl") } -include "security" { - path = find_in_parent_folders("dependencies/security.hcl") +dependency "security" { + config_path = "${get_original_terragrunt_dir()}/../security" + + mock_outputs = { + load_balancer_sg = "sg-00000000000000001" + api_vpc_link_sg = "sg-00000000000000002" + vpc_endpoint_sg = "sg-00000000000000003" + ecs_sg = "sg-00000000000000004" + runtime_sg = "sg-00000000000000005" + postgres_sg = "sg-00000000000000006" + } + + mock_outputs_merge_strategy_with_state = "shallow" + mock_outputs_allowed_terraform_commands = ["validate", "plan", "destroy", "init", "show", "graph-dependencies", "output-module-groups"] } terraform { @@ -18,7 +30,8 @@ dependency "cognito" { auth_issuer_url = "https://cognito-idp.eu-west-2.amazonaws.com/eu-west-2_mock" } - mock_outputs_allowed_terraform_commands = ["validate", "plan", "destroy", "init", "show"] + mock_outputs_merge_strategy_with_state = "shallow" + mock_outputs_allowed_terraform_commands = ["validate", "plan", "destroy", "init", "show", "graph-dependencies", "output-module-groups"] } inputs = { diff --git a/infra/live/dev/aws/observability/terragrunt.hcl b/infra/live/dev/aws/observability/terragrunt.hcl index 3a37c9ba..d3cae1c1 100644 --- a/infra/live/dev/aws/observability/terragrunt.hcl +++ b/infra/live/dev/aws/observability/terragrunt.hcl @@ -2,6 +2,10 @@ include "root" { path = find_in_parent_folders("root.hcl") } +dependencies { + paths = ["../oidc"] +} + terraform { source = "../../../../modules//aws//observability" } diff --git a/infra/live/dev/aws/rds_reader_tagger/terragrunt.hcl b/infra/live/dev/aws/rds_reader_tagger/terragrunt.hcl index bc618a77..2b177266 100644 --- a/infra/live/dev/aws/rds_reader_tagger/terragrunt.hcl +++ b/infra/live/dev/aws/rds_reader_tagger/terragrunt.hcl @@ -2,12 +2,41 @@ include "root" { path = find_in_parent_folders("root.hcl") } -locals { - database = read_terragrunt_config(find_in_parent_folders("dependencies/database.hcl")) +dependency "database" { + config_path = "${get_original_terragrunt_dir()}/../database" + + mock_outputs = { + database_credentials_secret_arn = "arn:aws:secretsmanager:eu-west-2:111111111111:secret:mock-database-credentials" + database_readwrite_endpoint = "mock-database.cluster-abcdefghijkl.eu-west-2.rds.amazonaws.com" + database_name = "app" + database_port = 5432 + database_cluster_identifier = "mock-database-cluster" + } + + mock_outputs_merge_strategy_with_state = "shallow" + mock_outputs_allowed_terraform_commands = ["validate", "plan", "destroy", "init", "show", "graph-dependencies", "output-module-groups"] +} + +dependency "code_bucket" { + config_path = "${get_original_terragrunt_dir()}/../code_bucket" + + mock_outputs = { + bucket = "mock-code-bucket" + } + + mock_outputs_merge_strategy_with_state = "shallow" + mock_outputs_allowed_terraform_commands = ["validate", "plan", "destroy", "init", "show", "graph-dependencies", "output-module-groups"] } terraform { source = "../../../../modules//aws//rds_reader_tagger" } -inputs = local.database.inputs +inputs = { + code_bucket = dependency.code_bucket.outputs.bucket + database_credentials_secret_arn = dependency.database.outputs.database_credentials_secret_arn + database_readwrite_endpoint = dependency.database.outputs.database_readwrite_endpoint + database_name = dependency.database.outputs.database_name + database_port = dependency.database.outputs.database_port + database_cluster_identifier = dependency.database.outputs.database_cluster_identifier +} diff --git a/infra/live/dev/aws/security/terragrunt.hcl b/infra/live/dev/aws/security/terragrunt.hcl index 7b418119..cecf3764 100644 --- a/infra/live/dev/aws/security/terragrunt.hcl +++ b/infra/live/dev/aws/security/terragrunt.hcl @@ -2,6 +2,10 @@ include "root" { path = find_in_parent_folders("root.hcl") } +dependencies { + paths = ["../oidc"] +} + terraform { source = "../../../../modules//aws//security" } diff --git a/infra/live/dev/aws/service_api/terragrunt.hcl b/infra/live/dev/aws/service_api/terragrunt.hcl index 501e7a44..f3f22b28 100644 --- a/infra/live/dev/aws/service_api/terragrunt.hcl +++ b/infra/live/dev/aws/service_api/terragrunt.hcl @@ -2,13 +2,71 @@ include "root" { path = find_in_parent_folders("root.hcl") } -include "security" { - path = find_in_parent_folders("dependencies/security.hcl") +dependencies { + paths = ["../database", "../messaging"] } -locals { - cluster = read_terragrunt_config(find_in_parent_folders("dependencies/cluster.hcl")) - network = read_terragrunt_config(find_in_parent_folders("dependencies/network.hcl")) +dependency "ecr" { + config_path = "${get_original_terragrunt_dir()}/../ecr" + + mock_outputs = { + repository_url = "111111111111.dkr.ecr.eu-west-2.amazonaws.com/mock-ecr" + repository_name = "mock-ecr" + repository_arn = "arn:aws:ecr:eu-west-2:111111111111:repository/mock-ecr" + } + + mock_outputs_merge_strategy_with_state = "shallow" + mock_outputs_allowed_terraform_commands = ["validate", "plan", "destroy", "init", "show", "graph-dependencies", "output-module-groups"] +} + +dependency "security" { + config_path = "${get_original_terragrunt_dir()}/../security" + + mock_outputs = { + load_balancer_sg = "sg-00000000000000001" + api_vpc_link_sg = "sg-00000000000000002" + vpc_endpoint_sg = "sg-00000000000000003" + ecs_sg = "sg-00000000000000004" + runtime_sg = "sg-00000000000000005" + postgres_sg = "sg-00000000000000006" + } + + mock_outputs_merge_strategy_with_state = "shallow" + mock_outputs_allowed_terraform_commands = ["validate", "plan", "destroy", "init", "show", "graph-dependencies", "output-module-groups"] +} + +dependency "cluster" { + config_path = "${get_original_terragrunt_dir()}/../cluster" + + mock_outputs = { + cluster_id = "mock-cluster-id" + cluster_name = "mock-cluster" + } + + mock_outputs_merge_strategy_with_state = "shallow" + mock_outputs_allowed_terraform_commands = ["validate", "plan", "destroy", "init", "show", "graph-dependencies", "output-module-groups"] +} + +dependency "network" { + config_path = "${get_original_terragrunt_dir()}/../network" + + mock_outputs = { + default_target_group_arn = "arn:aws:elasticloadbalancing:eu-west-2:111111111111:targetgroup/mock-default/1234567890abcdef" + load_balancer_arn = "arn:aws:elasticloadbalancing:eu-west-2:111111111111:loadbalancer/app/mock-internal/1234567890abcdef" + default_http_listener_arn = "arn:aws:elasticloadbalancing:eu-west-2:111111111111:listener/app/mock-internal/1234567890abcdef/abcdef1234567890" + load_balancer_arn_suffix = "app/mock-internal/1234567890abcdef" + target_group_arn_suffix = "targetgroup/mock-default/1234567890abcdef" + internal_invoke_url = "http://mock-internal-123456.eu-west-2.elb.amazonaws.com" + api_id = "mockapi123" + api_invoke_url = "https://mockapi123.execute-api.eu-west-2.amazonaws.com" + api_execution_arn = "arn:aws:execute-api:eu-west-2:111111111111:mockapi123" + api_stage_name = "$default" + vpc_link_id = "vpclink-mock123" + http_api_authorizer_id = "auth-mock123" + } + + mock_outputs_merge_strategy_with_state = "shallow" + mock_outputs_allowed_terraform_commands = ["validate", "plan", "destroy", "init", "show", "graph-dependencies", "output-module-groups"] } terraform { @@ -19,6 +77,22 @@ inputs = merge( { ecs_security_group_id = dependency.security.outputs.ecs_sg }, - local.cluster.inputs, - local.network.inputs, + { + cluster_id = dependency.cluster.outputs.cluster_id + cluster_name = dependency.cluster.outputs.cluster_name + }, + { + network_default_target_group_arn = dependency.network.outputs.default_target_group_arn + network_load_balancer_arn = dependency.network.outputs.load_balancer_arn + network_default_http_listener_arn = dependency.network.outputs.default_http_listener_arn + network_load_balancer_arn_suffix = dependency.network.outputs.load_balancer_arn_suffix + network_target_group_arn_suffix = dependency.network.outputs.target_group_arn_suffix + network_internal_invoke_url = dependency.network.outputs.internal_invoke_url + network_api_id = dependency.network.outputs.api_id + network_api_invoke_url = dependency.network.outputs.api_invoke_url + network_api_execution_arn = dependency.network.outputs.api_execution_arn + network_api_stage_name = dependency.network.outputs.api_stage_name + network_vpc_link_id = dependency.network.outputs.vpc_link_id + network_http_api_authorizer_id = dependency.network.outputs.http_api_authorizer_id + }, ) diff --git a/infra/live/dev/aws/service_worker/terragrunt.hcl b/infra/live/dev/aws/service_worker/terragrunt.hcl index 7b23580e..7970bb19 100644 --- a/infra/live/dev/aws/service_worker/terragrunt.hcl +++ b/infra/live/dev/aws/service_worker/terragrunt.hcl @@ -2,14 +2,93 @@ include "root" { path = find_in_parent_folders("root.hcl") } -include "security" { - path = find_in_parent_folders("dependencies/security.hcl") +dependencies { + paths = ["../database"] } -locals { - messaging = read_terragrunt_config(find_in_parent_folders("dependencies/messaging.hcl")) - cluster = read_terragrunt_config(find_in_parent_folders("dependencies/cluster.hcl")) - network = read_terragrunt_config(find_in_parent_folders("dependencies/network.hcl")) +dependency "ecr" { + config_path = "${get_original_terragrunt_dir()}/../ecr" + + mock_outputs = { + repository_url = "111111111111.dkr.ecr.eu-west-2.amazonaws.com/mock-ecr" + repository_name = "mock-ecr" + repository_arn = "arn:aws:ecr:eu-west-2:111111111111:repository/mock-ecr" + } + + mock_outputs_merge_strategy_with_state = "shallow" + mock_outputs_allowed_terraform_commands = ["validate", "plan", "destroy", "init", "show", "graph-dependencies", "output-module-groups"] +} + +dependency "security" { + config_path = "${get_original_terragrunt_dir()}/../security" + + mock_outputs = { + load_balancer_sg = "sg-00000000000000001" + api_vpc_link_sg = "sg-00000000000000002" + vpc_endpoint_sg = "sg-00000000000000003" + ecs_sg = "sg-00000000000000004" + runtime_sg = "sg-00000000000000005" + postgres_sg = "sg-00000000000000006" + } + + mock_outputs_merge_strategy_with_state = "shallow" + mock_outputs_allowed_terraform_commands = ["validate", "plan", "destroy", "init", "show", "graph-dependencies", "output-module-groups"] +} + +dependency "messaging" { + config_path = "${get_original_terragrunt_dir()}/../messaging" + + mock_outputs = { + worker_topic_name = "mock-worker-events" + worker_topic_arn = "arn:aws:sns:eu-west-2:111111111111:mock-worker-events" + worker_topic_publish_policy_arn = "arn:aws:iam::111111111111:policy/mock-worker-topic-publish" + lambda_worker_queue_name = "mock-lambda-worker-queue" + lambda_worker_queue_arn = "arn:aws:sqs:eu-west-2:111111111111:mock-lambda-worker-queue" + lambda_worker_queue_url = "https://sqs.eu-west-2.amazonaws.com/111111111111/mock-lambda-worker-queue" + lambda_worker_queue_read_policy_arn = "arn:aws:iam::111111111111:policy/mock-lambda-worker-queue-read" + lambda_worker_dead_letter_queue_name = "mock-lambda-worker-dlq" + lambda_worker_dead_letter_queue_url = "https://sqs.eu-west-2.amazonaws.com/111111111111/mock-lambda-worker-dlq" + ecs_worker_queue_name = "mock-ecs-worker-queue" + ecs_worker_queue_url = "https://sqs.eu-west-2.amazonaws.com/111111111111/mock-ecs-worker-queue" + ecs_worker_queue_read_policy_arn = "arn:aws:iam::111111111111:policy/mock-ecs-worker-queue-read" + } + + mock_outputs_merge_strategy_with_state = "shallow" + mock_outputs_allowed_terraform_commands = ["validate", "plan", "destroy", "init", "show", "graph-dependencies", "output-module-groups"] +} + +dependency "cluster" { + config_path = "${get_original_terragrunt_dir()}/../cluster" + + mock_outputs = { + cluster_id = "mock-cluster-id" + cluster_name = "mock-cluster" + } + + mock_outputs_merge_strategy_with_state = "shallow" + mock_outputs_allowed_terraform_commands = ["validate", "plan", "destroy", "init", "show", "graph-dependencies", "output-module-groups"] +} + +dependency "network" { + config_path = "${get_original_terragrunt_dir()}/../network" + + mock_outputs = { + default_target_group_arn = "arn:aws:elasticloadbalancing:eu-west-2:111111111111:targetgroup/mock-default/1234567890abcdef" + load_balancer_arn = "arn:aws:elasticloadbalancing:eu-west-2:111111111111:loadbalancer/app/mock-internal/1234567890abcdef" + default_http_listener_arn = "arn:aws:elasticloadbalancing:eu-west-2:111111111111:listener/app/mock-internal/1234567890abcdef/abcdef1234567890" + load_balancer_arn_suffix = "app/mock-internal/1234567890abcdef" + target_group_arn_suffix = "targetgroup/mock-default/1234567890abcdef" + internal_invoke_url = "http://mock-internal-123456.eu-west-2.elb.amazonaws.com" + api_id = "mockapi123" + api_invoke_url = "https://mockapi123.execute-api.eu-west-2.amazonaws.com" + api_execution_arn = "arn:aws:execute-api:eu-west-2:111111111111:mockapi123" + api_stage_name = "$default" + vpc_link_id = "vpclink-mock123" + http_api_authorizer_id = "auth-mock123" + } + + mock_outputs_merge_strategy_with_state = "shallow" + mock_outputs_allowed_terraform_commands = ["validate", "plan", "destroy", "init", "show", "graph-dependencies", "output-module-groups"] } terraform { @@ -20,7 +99,23 @@ inputs = merge( { ecs_security_group_id = dependency.security.outputs.ecs_sg }, - local.messaging.inputs, - local.cluster.inputs, - local.network.inputs, + dependency.messaging.outputs, + { + cluster_id = dependency.cluster.outputs.cluster_id + cluster_name = dependency.cluster.outputs.cluster_name + }, + { + network_default_target_group_arn = dependency.network.outputs.default_target_group_arn + network_load_balancer_arn = dependency.network.outputs.load_balancer_arn + network_default_http_listener_arn = dependency.network.outputs.default_http_listener_arn + network_load_balancer_arn_suffix = dependency.network.outputs.load_balancer_arn_suffix + network_target_group_arn_suffix = dependency.network.outputs.target_group_arn_suffix + network_internal_invoke_url = dependency.network.outputs.internal_invoke_url + network_api_id = dependency.network.outputs.api_id + network_api_invoke_url = dependency.network.outputs.api_invoke_url + network_api_execution_arn = dependency.network.outputs.api_execution_arn + network_api_stage_name = dependency.network.outputs.api_stage_name + network_vpc_link_id = dependency.network.outputs.vpc_link_id + network_http_api_authorizer_id = dependency.network.outputs.http_api_authorizer_id + }, ) diff --git a/infra/live/dev/aws/task_api/terragrunt.hcl b/infra/live/dev/aws/task_api/terragrunt.hcl index af324e86..0d192fda 100644 --- a/infra/live/dev/aws/task_api/terragrunt.hcl +++ b/infra/live/dev/aws/task_api/terragrunt.hcl @@ -2,6 +2,10 @@ include "root" { path = find_in_parent_folders("root.hcl") } +dependencies { + paths = ["../security", "../cluster", "../network"] +} + terraform { source = "../../../../modules//aws//task_api" } diff --git a/infra/live/dev/aws/task_worker/terragrunt.hcl b/infra/live/dev/aws/task_worker/terragrunt.hcl index b28cbca9..d9580640 100644 --- a/infra/live/dev/aws/task_worker/terragrunt.hcl +++ b/infra/live/dev/aws/task_worker/terragrunt.hcl @@ -2,13 +2,58 @@ include "root" { path = find_in_parent_folders("root.hcl") } -locals { - messaging = read_terragrunt_config(find_in_parent_folders("dependencies/messaging.hcl")) - database = read_terragrunt_config(find_in_parent_folders("dependencies/database.hcl")) +dependencies { + paths = ["../security", "../network"] +} + +dependency "messaging" { + config_path = "${get_original_terragrunt_dir()}/../messaging" + + mock_outputs = { + worker_topic_name = "mock-worker-events" + worker_topic_arn = "arn:aws:sns:eu-west-2:111111111111:mock-worker-events" + worker_topic_publish_policy_arn = "arn:aws:iam::111111111111:policy/mock-worker-topic-publish" + lambda_worker_queue_name = "mock-lambda-worker-queue" + lambda_worker_queue_arn = "arn:aws:sqs:eu-west-2:111111111111:mock-lambda-worker-queue" + lambda_worker_queue_url = "https://sqs.eu-west-2.amazonaws.com/111111111111/mock-lambda-worker-queue" + lambda_worker_queue_read_policy_arn = "arn:aws:iam::111111111111:policy/mock-lambda-worker-queue-read" + lambda_worker_dead_letter_queue_name = "mock-lambda-worker-dlq" + lambda_worker_dead_letter_queue_url = "https://sqs.eu-west-2.amazonaws.com/111111111111/mock-lambda-worker-dlq" + ecs_worker_queue_name = "mock-ecs-worker-queue" + ecs_worker_queue_url = "https://sqs.eu-west-2.amazonaws.com/111111111111/mock-ecs-worker-queue" + ecs_worker_queue_read_policy_arn = "arn:aws:iam::111111111111:policy/mock-ecs-worker-queue-read" + } + + mock_outputs_merge_strategy_with_state = "shallow" + mock_outputs_allowed_terraform_commands = ["validate", "plan", "destroy", "init", "show", "graph-dependencies", "output-module-groups"] +} + +dependency "database" { + config_path = "${get_original_terragrunt_dir()}/../database" + + mock_outputs = { + database_credentials_secret_arn = "arn:aws:secretsmanager:eu-west-2:111111111111:secret:mock-database-credentials" + database_readwrite_endpoint = "mock-database.cluster-abcdefghijkl.eu-west-2.rds.amazonaws.com" + database_name = "app" + database_port = 5432 + database_cluster_identifier = "mock-database-cluster" + } + + mock_outputs_merge_strategy_with_state = "shallow" + mock_outputs_allowed_terraform_commands = ["validate", "plan", "destroy", "init", "show", "graph-dependencies", "output-module-groups"] } terraform { source = "../../../../modules//aws//task_worker" } -inputs = merge(local.messaging.inputs, local.database.inputs) +inputs = merge( + dependency.messaging.outputs, + { + database_credentials_secret_arn = dependency.database.outputs.database_credentials_secret_arn + database_readwrite_endpoint = dependency.database.outputs.database_readwrite_endpoint + database_name = dependency.database.outputs.database_name + database_port = dependency.database.outputs.database_port + database_cluster_identifier = dependency.database.outputs.database_cluster_identifier + }, +) diff --git a/infra/live/global_vars.hcl b/infra/live/global_vars.hcl index 4c04287b..bfef183d 100644 --- a/infra/live/global_vars.hcl +++ b/infra/live/global_vars.hcl @@ -1,6 +1,7 @@ locals { - vpc_name = "vpc" - aws_region = "eu-west-2" + vpc_name = "vpc" + aws_region = "eu-west-2" + domain_name = "chrispsheehan.com" allowed_role_actions = [ "s3:*", "iam:*", @@ -35,6 +36,7 @@ locals { inputs = { vpc_name = local.vpc_name aws_region = local.aws_region + domain_name = local.domain_name allowed_role_actions = local.allowed_role_actions code_artifact_expiration_days = local.code_artifact_expiration_days infra_plan_artifact_expiration_days = local.infra_plan_artifact_expiration_days diff --git a/infra/live/prod/aws/cluster/terragrunt.hcl b/infra/live/prod/aws/cluster/terragrunt.hcl index 8d29bd5d..b52fb59a 100644 --- a/infra/live/prod/aws/cluster/terragrunt.hcl +++ b/infra/live/prod/aws/cluster/terragrunt.hcl @@ -2,6 +2,10 @@ include "root" { path = find_in_parent_folders("root.hcl") } +dependencies { + paths = ["../oidc"] +} + terraform { source = "../../../../modules//aws//_shared//cluster" } diff --git a/infra/live/prod/aws/cognito/terragrunt.hcl b/infra/live/prod/aws/cognito/terragrunt.hcl index 6e2da3c4..5f937331 100644 --- a/infra/live/prod/aws/cognito/terragrunt.hcl +++ b/infra/live/prod/aws/cognito/terragrunt.hcl @@ -2,6 +2,10 @@ include "root" { path = find_in_parent_folders("root.hcl") } +dependencies { + paths = ["../oidc"] +} + terraform { source = "../../../../modules//aws//cognito" } diff --git a/infra/live/prod/aws/database/terragrunt.hcl b/infra/live/prod/aws/database/terragrunt.hcl index 5901ba70..8f8f5a6e 100644 --- a/infra/live/prod/aws/database/terragrunt.hcl +++ b/infra/live/prod/aws/database/terragrunt.hcl @@ -2,8 +2,20 @@ include "root" { path = find_in_parent_folders("root.hcl") } -include "security" { - path = find_in_parent_folders("dependencies/security.hcl") +dependency "security" { + config_path = "${get_original_terragrunt_dir()}/../security" + + mock_outputs = { + load_balancer_sg = "sg-00000000000000001" + api_vpc_link_sg = "sg-00000000000000002" + vpc_endpoint_sg = "sg-00000000000000003" + ecs_sg = "sg-00000000000000004" + runtime_sg = "sg-00000000000000005" + postgres_sg = "sg-00000000000000006" + } + + mock_outputs_merge_strategy_with_state = "shallow" + mock_outputs_allowed_terraform_commands = ["validate", "plan", "destroy", "init", "show", "graph-dependencies", "output-module-groups"] } terraform { diff --git a/infra/live/prod/aws/frontend/terragrunt.hcl b/infra/live/prod/aws/frontend/terragrunt.hcl index fa22a838..bc2220f9 100644 --- a/infra/live/prod/aws/frontend/terragrunt.hcl +++ b/infra/live/prod/aws/frontend/terragrunt.hcl @@ -2,12 +2,39 @@ include "root" { path = find_in_parent_folders("root.hcl") } -locals { - frontend = read_terragrunt_config(find_in_parent_folders("dependencies/frontend.hcl")) +dependency "network" { + config_path = "${get_original_terragrunt_dir()}/../network" + + mock_outputs = { + api_invoke_url = "https://mockapi123.execute-api.eu-west-2.amazonaws.com" + } + + mock_outputs_merge_strategy_with_state = "shallow" + mock_outputs_allowed_terraform_commands = ["validate", "plan", "destroy", "init", "show", "graph-dependencies", "output-module-groups"] +} + +dependency "cognito" { + config_path = "${get_original_terragrunt_dir()}/../cognito" + + mock_outputs = { + auth_user_pool_id = "eu-west-2_mock" + auth_user_pool_client_id = "mock-user-pool-client-id" + auth_hosted_ui_url = "https://mock-domain.auth.eu-west-2.amazoncognito.com" + auth_readonly_group_name = "readonly" + } + + mock_outputs_merge_strategy_with_state = "shallow" + mock_outputs_allowed_terraform_commands = ["validate", "plan", "destroy", "init", "show", "graph-dependencies", "output-module-groups"] } terraform { source = "../../../../modules//aws//frontend" } -inputs = local.frontend.inputs +inputs = { + api_invoke_url = dependency.network.outputs.api_invoke_url + auth_user_pool_id = dependency.cognito.outputs.auth_user_pool_id + auth_user_pool_client_id = dependency.cognito.outputs.auth_user_pool_client_id + auth_hosted_ui_url = dependency.cognito.outputs.auth_hosted_ui_url + auth_readonly_group_name = dependency.cognito.outputs.auth_readonly_group_name +} diff --git a/infra/live/prod/aws/lambda_api/terragrunt.hcl b/infra/live/prod/aws/lambda_api/terragrunt.hcl index 8e29bf96..dfc53d7d 100644 --- a/infra/live/prod/aws/lambda_api/terragrunt.hcl +++ b/infra/live/prod/aws/lambda_api/terragrunt.hcl @@ -2,9 +2,52 @@ include "root" { path = find_in_parent_folders("root.hcl") } -locals { - network = read_terragrunt_config(find_in_parent_folders("dependencies/network.hcl")) - messaging = read_terragrunt_config(find_in_parent_folders("dependencies/messaging.hcl")) +dependencies { + paths = ["../security", "../database"] +} + +dependency "network" { + config_path = "${get_original_terragrunt_dir()}/../network" + + mock_outputs = { + default_target_group_arn = "arn:aws:elasticloadbalancing:eu-west-2:111111111111:targetgroup/mock-default/1234567890abcdef" + load_balancer_arn = "arn:aws:elasticloadbalancing:eu-west-2:111111111111:loadbalancer/app/mock-internal/1234567890abcdef" + default_http_listener_arn = "arn:aws:elasticloadbalancing:eu-west-2:111111111111:listener/app/mock-internal/1234567890abcdef/abcdef1234567890" + load_balancer_arn_suffix = "app/mock-internal/1234567890abcdef" + target_group_arn_suffix = "targetgroup/mock-default/1234567890abcdef" + internal_invoke_url = "http://mock-internal-123456.eu-west-2.elb.amazonaws.com" + api_id = "mockapi123" + api_invoke_url = "https://mockapi123.execute-api.eu-west-2.amazonaws.com" + api_execution_arn = "arn:aws:execute-api:eu-west-2:111111111111:mockapi123" + api_stage_name = "$default" + vpc_link_id = "vpclink-mock123" + http_api_authorizer_id = "auth-mock123" + } + + mock_outputs_merge_strategy_with_state = "shallow" + mock_outputs_allowed_terraform_commands = ["validate", "plan", "destroy", "init", "show", "graph-dependencies", "output-module-groups"] +} + +dependency "messaging" { + config_path = "${get_original_terragrunt_dir()}/../messaging" + + mock_outputs = { + worker_topic_name = "mock-worker-events" + worker_topic_arn = "arn:aws:sns:eu-west-2:111111111111:mock-worker-events" + worker_topic_publish_policy_arn = "arn:aws:iam::111111111111:policy/mock-worker-topic-publish" + lambda_worker_queue_name = "mock-lambda-worker-queue" + lambda_worker_queue_arn = "arn:aws:sqs:eu-west-2:111111111111:mock-lambda-worker-queue" + lambda_worker_queue_url = "https://sqs.eu-west-2.amazonaws.com/111111111111/mock-lambda-worker-queue" + lambda_worker_queue_read_policy_arn = "arn:aws:iam::111111111111:policy/mock-lambda-worker-queue-read" + lambda_worker_dead_letter_queue_name = "mock-lambda-worker-dlq" + lambda_worker_dead_letter_queue_url = "https://sqs.eu-west-2.amazonaws.com/111111111111/mock-lambda-worker-dlq" + ecs_worker_queue_name = "mock-ecs-worker-queue" + ecs_worker_queue_url = "https://sqs.eu-west-2.amazonaws.com/111111111111/mock-ecs-worker-queue" + ecs_worker_queue_read_policy_arn = "arn:aws:iam::111111111111:policy/mock-ecs-worker-queue-read" + } + + mock_outputs_merge_strategy_with_state = "shallow" + mock_outputs_allowed_terraform_commands = ["validate", "plan", "destroy", "init", "show", "graph-dependencies", "output-module-groups"] } terraform { @@ -12,8 +55,21 @@ terraform { } inputs = merge( - local.network.inputs, - local.messaging.inputs, + { + network_default_target_group_arn = dependency.network.outputs.default_target_group_arn + network_load_balancer_arn = dependency.network.outputs.load_balancer_arn + network_default_http_listener_arn = dependency.network.outputs.default_http_listener_arn + network_load_balancer_arn_suffix = dependency.network.outputs.load_balancer_arn_suffix + network_target_group_arn_suffix = dependency.network.outputs.target_group_arn_suffix + network_internal_invoke_url = dependency.network.outputs.internal_invoke_url + network_api_id = dependency.network.outputs.api_id + network_api_invoke_url = dependency.network.outputs.api_invoke_url + network_api_execution_arn = dependency.network.outputs.api_execution_arn + network_api_stage_name = dependency.network.outputs.api_stage_name + network_vpc_link_id = dependency.network.outputs.vpc_link_id + network_http_api_authorizer_id = dependency.network.outputs.http_api_authorizer_id + }, + dependency.messaging.outputs, { api_5xx_alarm_threshold = 5.0 api_5xx_alarm_evaluation_periods = 3 diff --git a/infra/live/prod/aws/lambda_worker/terragrunt.hcl b/infra/live/prod/aws/lambda_worker/terragrunt.hcl index 294342f8..9ba0a4c3 100644 --- a/infra/live/prod/aws/lambda_worker/terragrunt.hcl +++ b/infra/live/prod/aws/lambda_worker/terragrunt.hcl @@ -2,8 +2,30 @@ include "root" { path = find_in_parent_folders("root.hcl") } -locals { - messaging = read_terragrunt_config(find_in_parent_folders("dependencies/messaging.hcl")) +dependencies { + paths = ["../security", "../network", "../database"] +} + +dependency "messaging" { + config_path = "${get_original_terragrunt_dir()}/../messaging" + + mock_outputs = { + worker_topic_name = "mock-worker-events" + worker_topic_arn = "arn:aws:sns:eu-west-2:111111111111:mock-worker-events" + worker_topic_publish_policy_arn = "arn:aws:iam::111111111111:policy/mock-worker-topic-publish" + lambda_worker_queue_name = "mock-lambda-worker-queue" + lambda_worker_queue_arn = "arn:aws:sqs:eu-west-2:111111111111:mock-lambda-worker-queue" + lambda_worker_queue_url = "https://sqs.eu-west-2.amazonaws.com/111111111111/mock-lambda-worker-queue" + lambda_worker_queue_read_policy_arn = "arn:aws:iam::111111111111:policy/mock-lambda-worker-queue-read" + lambda_worker_dead_letter_queue_name = "mock-lambda-worker-dlq" + lambda_worker_dead_letter_queue_url = "https://sqs.eu-west-2.amazonaws.com/111111111111/mock-lambda-worker-dlq" + ecs_worker_queue_name = "mock-ecs-worker-queue" + ecs_worker_queue_url = "https://sqs.eu-west-2.amazonaws.com/111111111111/mock-ecs-worker-queue" + ecs_worker_queue_read_policy_arn = "arn:aws:iam::111111111111:policy/mock-ecs-worker-queue-read" + } + + mock_outputs_merge_strategy_with_state = "shallow" + mock_outputs_allowed_terraform_commands = ["validate", "plan", "destroy", "init", "show", "graph-dependencies", "output-module-groups"] } terraform { @@ -11,7 +33,7 @@ terraform { } inputs = merge( - local.messaging.inputs, + dependency.messaging.outputs, { sqs_dlq_alarm_threshold = 5 # fail when there are 5 messages in the DLQ sqs_dlq_alarm_evaluation_periods = 3 diff --git a/infra/live/prod/aws/messaging/terragrunt.hcl b/infra/live/prod/aws/messaging/terragrunt.hcl index b85cf717..8cf46acc 100644 --- a/infra/live/prod/aws/messaging/terragrunt.hcl +++ b/infra/live/prod/aws/messaging/terragrunt.hcl @@ -2,6 +2,10 @@ include "root" { path = find_in_parent_folders("root.hcl") } +dependencies { + paths = ["../oidc"] +} + terraform { source = "../../../../modules//aws//messaging" } diff --git a/infra/live/prod/aws/migrations/terragrunt.hcl b/infra/live/prod/aws/migrations/terragrunt.hcl index edc1b7f2..857435f6 100644 --- a/infra/live/prod/aws/migrations/terragrunt.hcl +++ b/infra/live/prod/aws/migrations/terragrunt.hcl @@ -2,12 +2,35 @@ include "root" { path = find_in_parent_folders("root.hcl") } -include "security" { - path = find_in_parent_folders("dependencies/security.hcl") +dependency "security" { + config_path = "${get_original_terragrunt_dir()}/../security" + + mock_outputs = { + load_balancer_sg = "sg-00000000000000001" + api_vpc_link_sg = "sg-00000000000000002" + vpc_endpoint_sg = "sg-00000000000000003" + ecs_sg = "sg-00000000000000004" + runtime_sg = "sg-00000000000000005" + postgres_sg = "sg-00000000000000006" + } + + mock_outputs_merge_strategy_with_state = "shallow" + mock_outputs_allowed_terraform_commands = ["validate", "plan", "destroy", "init", "show", "graph-dependencies", "output-module-groups"] } -locals { - database = read_terragrunt_config(find_in_parent_folders("dependencies/database.hcl")) +dependency "database" { + config_path = "${get_original_terragrunt_dir()}/../database" + + mock_outputs = { + database_credentials_secret_arn = "arn:aws:secretsmanager:eu-west-2:111111111111:secret:mock-database-credentials" + database_readwrite_endpoint = "mock-database.cluster-abcdefghijkl.eu-west-2.rds.amazonaws.com" + database_name = "app" + database_port = 5432 + database_cluster_identifier = "mock-database-cluster" + } + + mock_outputs_merge_strategy_with_state = "shallow" + mock_outputs_allowed_terraform_commands = ["validate", "plan", "destroy", "init", "show", "graph-dependencies", "output-module-groups"] } terraform { @@ -18,5 +41,11 @@ inputs = merge( { runtime_security_group_id = dependency.security.outputs.runtime_sg }, - local.database.inputs, + { + database_credentials_secret_arn = dependency.database.outputs.database_credentials_secret_arn + database_readwrite_endpoint = dependency.database.outputs.database_readwrite_endpoint + database_name = dependency.database.outputs.database_name + database_port = dependency.database.outputs.database_port + database_cluster_identifier = dependency.database.outputs.database_cluster_identifier + }, ) diff --git a/infra/live/prod/aws/network/terragrunt.hcl b/infra/live/prod/aws/network/terragrunt.hcl index 93443bf3..0824c6dc 100644 --- a/infra/live/prod/aws/network/terragrunt.hcl +++ b/infra/live/prod/aws/network/terragrunt.hcl @@ -2,8 +2,20 @@ include "root" { path = find_in_parent_folders("root.hcl") } -include "security" { - path = find_in_parent_folders("dependencies/security.hcl") +dependency "security" { + config_path = "${get_original_terragrunt_dir()}/../security" + + mock_outputs = { + load_balancer_sg = "sg-00000000000000001" + api_vpc_link_sg = "sg-00000000000000002" + vpc_endpoint_sg = "sg-00000000000000003" + ecs_sg = "sg-00000000000000004" + runtime_sg = "sg-00000000000000005" + postgres_sg = "sg-00000000000000006" + } + + mock_outputs_merge_strategy_with_state = "shallow" + mock_outputs_allowed_terraform_commands = ["validate", "plan", "destroy", "init", "show", "graph-dependencies", "output-module-groups"] } terraform { @@ -18,7 +30,8 @@ dependency "cognito" { auth_issuer_url = "https://cognito-idp.eu-west-2.amazonaws.com/eu-west-2_mock" } - mock_outputs_allowed_terraform_commands = ["validate", "plan", "destroy", "init", "show"] + mock_outputs_merge_strategy_with_state = "shallow" + mock_outputs_allowed_terraform_commands = ["validate", "plan", "destroy", "init", "show", "graph-dependencies", "output-module-groups"] } inputs = { diff --git a/infra/live/prod/aws/observability/terragrunt.hcl b/infra/live/prod/aws/observability/terragrunt.hcl index 3a37c9ba..d3cae1c1 100644 --- a/infra/live/prod/aws/observability/terragrunt.hcl +++ b/infra/live/prod/aws/observability/terragrunt.hcl @@ -2,6 +2,10 @@ include "root" { path = find_in_parent_folders("root.hcl") } +dependencies { + paths = ["../oidc"] +} + terraform { source = "../../../../modules//aws//observability" } diff --git a/infra/live/prod/aws/rds_reader_tagger/terragrunt.hcl b/infra/live/prod/aws/rds_reader_tagger/terragrunt.hcl index bc618a77..8eda104e 100644 --- a/infra/live/prod/aws/rds_reader_tagger/terragrunt.hcl +++ b/infra/live/prod/aws/rds_reader_tagger/terragrunt.hcl @@ -2,12 +2,29 @@ include "root" { path = find_in_parent_folders("root.hcl") } -locals { - database = read_terragrunt_config(find_in_parent_folders("dependencies/database.hcl")) +dependency "database" { + config_path = "${get_original_terragrunt_dir()}/../database" + + mock_outputs = { + database_credentials_secret_arn = "arn:aws:secretsmanager:eu-west-2:111111111111:secret:mock-database-credentials" + database_readwrite_endpoint = "mock-database.cluster-abcdefghijkl.eu-west-2.rds.amazonaws.com" + database_name = "app" + database_port = 5432 + database_cluster_identifier = "mock-database-cluster" + } + + mock_outputs_merge_strategy_with_state = "shallow" + mock_outputs_allowed_terraform_commands = ["validate", "plan", "destroy", "init", "show", "graph-dependencies", "output-module-groups"] } terraform { source = "../../../../modules//aws//rds_reader_tagger" } -inputs = local.database.inputs +inputs = { + database_credentials_secret_arn = dependency.database.outputs.database_credentials_secret_arn + database_readwrite_endpoint = dependency.database.outputs.database_readwrite_endpoint + database_name = dependency.database.outputs.database_name + database_port = dependency.database.outputs.database_port + database_cluster_identifier = dependency.database.outputs.database_cluster_identifier +} diff --git a/infra/live/prod/aws/security/terragrunt.hcl b/infra/live/prod/aws/security/terragrunt.hcl index 7b418119..cecf3764 100644 --- a/infra/live/prod/aws/security/terragrunt.hcl +++ b/infra/live/prod/aws/security/terragrunt.hcl @@ -2,6 +2,10 @@ include "root" { path = find_in_parent_folders("root.hcl") } +dependencies { + paths = ["../oidc"] +} + terraform { source = "../../../../modules//aws//security" } diff --git a/infra/live/prod/aws/service_api/terragrunt.hcl b/infra/live/prod/aws/service_api/terragrunt.hcl index 501e7a44..7e898bda 100644 --- a/infra/live/prod/aws/service_api/terragrunt.hcl +++ b/infra/live/prod/aws/service_api/terragrunt.hcl @@ -2,13 +2,58 @@ include "root" { path = find_in_parent_folders("root.hcl") } -include "security" { - path = find_in_parent_folders("dependencies/security.hcl") +dependencies { + paths = ["../database", "../messaging"] } -locals { - cluster = read_terragrunt_config(find_in_parent_folders("dependencies/cluster.hcl")) - network = read_terragrunt_config(find_in_parent_folders("dependencies/network.hcl")) +dependency "security" { + config_path = "${get_original_terragrunt_dir()}/../security" + + mock_outputs = { + load_balancer_sg = "sg-00000000000000001" + api_vpc_link_sg = "sg-00000000000000002" + vpc_endpoint_sg = "sg-00000000000000003" + ecs_sg = "sg-00000000000000004" + runtime_sg = "sg-00000000000000005" + postgres_sg = "sg-00000000000000006" + } + + mock_outputs_merge_strategy_with_state = "shallow" + mock_outputs_allowed_terraform_commands = ["validate", "plan", "destroy", "init", "show", "graph-dependencies", "output-module-groups"] +} + +dependency "cluster" { + config_path = "${get_original_terragrunt_dir()}/../cluster" + + mock_outputs = { + cluster_id = "mock-cluster-id" + cluster_name = "mock-cluster" + } + + mock_outputs_merge_strategy_with_state = "shallow" + mock_outputs_allowed_terraform_commands = ["validate", "plan", "destroy", "init", "show", "graph-dependencies", "output-module-groups"] +} + +dependency "network" { + config_path = "${get_original_terragrunt_dir()}/../network" + + mock_outputs = { + default_target_group_arn = "arn:aws:elasticloadbalancing:eu-west-2:111111111111:targetgroup/mock-default/1234567890abcdef" + load_balancer_arn = "arn:aws:elasticloadbalancing:eu-west-2:111111111111:loadbalancer/app/mock-internal/1234567890abcdef" + default_http_listener_arn = "arn:aws:elasticloadbalancing:eu-west-2:111111111111:listener/app/mock-internal/1234567890abcdef/abcdef1234567890" + load_balancer_arn_suffix = "app/mock-internal/1234567890abcdef" + target_group_arn_suffix = "targetgroup/mock-default/1234567890abcdef" + internal_invoke_url = "http://mock-internal-123456.eu-west-2.elb.amazonaws.com" + api_id = "mockapi123" + api_invoke_url = "https://mockapi123.execute-api.eu-west-2.amazonaws.com" + api_execution_arn = "arn:aws:execute-api:eu-west-2:111111111111:mockapi123" + api_stage_name = "$default" + vpc_link_id = "vpclink-mock123" + http_api_authorizer_id = "auth-mock123" + } + + mock_outputs_merge_strategy_with_state = "shallow" + mock_outputs_allowed_terraform_commands = ["validate", "plan", "destroy", "init", "show", "graph-dependencies", "output-module-groups"] } terraform { @@ -19,6 +64,22 @@ inputs = merge( { ecs_security_group_id = dependency.security.outputs.ecs_sg }, - local.cluster.inputs, - local.network.inputs, + { + cluster_id = dependency.cluster.outputs.cluster_id + cluster_name = dependency.cluster.outputs.cluster_name + }, + { + network_default_target_group_arn = dependency.network.outputs.default_target_group_arn + network_load_balancer_arn = dependency.network.outputs.load_balancer_arn + network_default_http_listener_arn = dependency.network.outputs.default_http_listener_arn + network_load_balancer_arn_suffix = dependency.network.outputs.load_balancer_arn_suffix + network_target_group_arn_suffix = dependency.network.outputs.target_group_arn_suffix + network_internal_invoke_url = dependency.network.outputs.internal_invoke_url + network_api_id = dependency.network.outputs.api_id + network_api_invoke_url = dependency.network.outputs.api_invoke_url + network_api_execution_arn = dependency.network.outputs.api_execution_arn + network_api_stage_name = dependency.network.outputs.api_stage_name + network_vpc_link_id = dependency.network.outputs.vpc_link_id + network_http_api_authorizer_id = dependency.network.outputs.http_api_authorizer_id + }, ) diff --git a/infra/live/prod/aws/service_worker/terragrunt.hcl b/infra/live/prod/aws/service_worker/terragrunt.hcl index 7b23580e..f7ec20d6 100644 --- a/infra/live/prod/aws/service_worker/terragrunt.hcl +++ b/infra/live/prod/aws/service_worker/terragrunt.hcl @@ -2,14 +2,80 @@ include "root" { path = find_in_parent_folders("root.hcl") } -include "security" { - path = find_in_parent_folders("dependencies/security.hcl") +dependencies { + paths = ["../database"] } -locals { - messaging = read_terragrunt_config(find_in_parent_folders("dependencies/messaging.hcl")) - cluster = read_terragrunt_config(find_in_parent_folders("dependencies/cluster.hcl")) - network = read_terragrunt_config(find_in_parent_folders("dependencies/network.hcl")) +dependency "security" { + config_path = "${get_original_terragrunt_dir()}/../security" + + mock_outputs = { + load_balancer_sg = "sg-00000000000000001" + api_vpc_link_sg = "sg-00000000000000002" + vpc_endpoint_sg = "sg-00000000000000003" + ecs_sg = "sg-00000000000000004" + runtime_sg = "sg-00000000000000005" + postgres_sg = "sg-00000000000000006" + } + + mock_outputs_merge_strategy_with_state = "shallow" + mock_outputs_allowed_terraform_commands = ["validate", "plan", "destroy", "init", "show", "graph-dependencies", "output-module-groups"] +} + +dependency "messaging" { + config_path = "${get_original_terragrunt_dir()}/../messaging" + + mock_outputs = { + worker_topic_name = "mock-worker-events" + worker_topic_arn = "arn:aws:sns:eu-west-2:111111111111:mock-worker-events" + worker_topic_publish_policy_arn = "arn:aws:iam::111111111111:policy/mock-worker-topic-publish" + lambda_worker_queue_name = "mock-lambda-worker-queue" + lambda_worker_queue_arn = "arn:aws:sqs:eu-west-2:111111111111:mock-lambda-worker-queue" + lambda_worker_queue_url = "https://sqs.eu-west-2.amazonaws.com/111111111111/mock-lambda-worker-queue" + lambda_worker_queue_read_policy_arn = "arn:aws:iam::111111111111:policy/mock-lambda-worker-queue-read" + lambda_worker_dead_letter_queue_name = "mock-lambda-worker-dlq" + lambda_worker_dead_letter_queue_url = "https://sqs.eu-west-2.amazonaws.com/111111111111/mock-lambda-worker-dlq" + ecs_worker_queue_name = "mock-ecs-worker-queue" + ecs_worker_queue_url = "https://sqs.eu-west-2.amazonaws.com/111111111111/mock-ecs-worker-queue" + ecs_worker_queue_read_policy_arn = "arn:aws:iam::111111111111:policy/mock-ecs-worker-queue-read" + } + + mock_outputs_merge_strategy_with_state = "shallow" + mock_outputs_allowed_terraform_commands = ["validate", "plan", "destroy", "init", "show", "graph-dependencies", "output-module-groups"] +} + +dependency "cluster" { + config_path = "${get_original_terragrunt_dir()}/../cluster" + + mock_outputs = { + cluster_id = "mock-cluster-id" + cluster_name = "mock-cluster" + } + + mock_outputs_merge_strategy_with_state = "shallow" + mock_outputs_allowed_terraform_commands = ["validate", "plan", "destroy", "init", "show", "graph-dependencies", "output-module-groups"] +} + +dependency "network" { + config_path = "${get_original_terragrunt_dir()}/../network" + + mock_outputs = { + default_target_group_arn = "arn:aws:elasticloadbalancing:eu-west-2:111111111111:targetgroup/mock-default/1234567890abcdef" + load_balancer_arn = "arn:aws:elasticloadbalancing:eu-west-2:111111111111:loadbalancer/app/mock-internal/1234567890abcdef" + default_http_listener_arn = "arn:aws:elasticloadbalancing:eu-west-2:111111111111:listener/app/mock-internal/1234567890abcdef/abcdef1234567890" + load_balancer_arn_suffix = "app/mock-internal/1234567890abcdef" + target_group_arn_suffix = "targetgroup/mock-default/1234567890abcdef" + internal_invoke_url = "http://mock-internal-123456.eu-west-2.elb.amazonaws.com" + api_id = "mockapi123" + api_invoke_url = "https://mockapi123.execute-api.eu-west-2.amazonaws.com" + api_execution_arn = "arn:aws:execute-api:eu-west-2:111111111111:mockapi123" + api_stage_name = "$default" + vpc_link_id = "vpclink-mock123" + http_api_authorizer_id = "auth-mock123" + } + + mock_outputs_merge_strategy_with_state = "shallow" + mock_outputs_allowed_terraform_commands = ["validate", "plan", "destroy", "init", "show", "graph-dependencies", "output-module-groups"] } terraform { @@ -20,7 +86,23 @@ inputs = merge( { ecs_security_group_id = dependency.security.outputs.ecs_sg }, - local.messaging.inputs, - local.cluster.inputs, - local.network.inputs, + dependency.messaging.outputs, + { + cluster_id = dependency.cluster.outputs.cluster_id + cluster_name = dependency.cluster.outputs.cluster_name + }, + { + network_default_target_group_arn = dependency.network.outputs.default_target_group_arn + network_load_balancer_arn = dependency.network.outputs.load_balancer_arn + network_default_http_listener_arn = dependency.network.outputs.default_http_listener_arn + network_load_balancer_arn_suffix = dependency.network.outputs.load_balancer_arn_suffix + network_target_group_arn_suffix = dependency.network.outputs.target_group_arn_suffix + network_internal_invoke_url = dependency.network.outputs.internal_invoke_url + network_api_id = dependency.network.outputs.api_id + network_api_invoke_url = dependency.network.outputs.api_invoke_url + network_api_execution_arn = dependency.network.outputs.api_execution_arn + network_api_stage_name = dependency.network.outputs.api_stage_name + network_vpc_link_id = dependency.network.outputs.vpc_link_id + network_http_api_authorizer_id = dependency.network.outputs.http_api_authorizer_id + }, ) diff --git a/infra/live/prod/aws/task_api/terragrunt.hcl b/infra/live/prod/aws/task_api/terragrunt.hcl index af324e86..0d192fda 100644 --- a/infra/live/prod/aws/task_api/terragrunt.hcl +++ b/infra/live/prod/aws/task_api/terragrunt.hcl @@ -2,6 +2,10 @@ include "root" { path = find_in_parent_folders("root.hcl") } +dependencies { + paths = ["../security", "../cluster", "../network"] +} + terraform { source = "../../../../modules//aws//task_api" } diff --git a/infra/live/prod/aws/task_worker/terragrunt.hcl b/infra/live/prod/aws/task_worker/terragrunt.hcl index b28cbca9..d9580640 100644 --- a/infra/live/prod/aws/task_worker/terragrunt.hcl +++ b/infra/live/prod/aws/task_worker/terragrunt.hcl @@ -2,13 +2,58 @@ include "root" { path = find_in_parent_folders("root.hcl") } -locals { - messaging = read_terragrunt_config(find_in_parent_folders("dependencies/messaging.hcl")) - database = read_terragrunt_config(find_in_parent_folders("dependencies/database.hcl")) +dependencies { + paths = ["../security", "../network"] +} + +dependency "messaging" { + config_path = "${get_original_terragrunt_dir()}/../messaging" + + mock_outputs = { + worker_topic_name = "mock-worker-events" + worker_topic_arn = "arn:aws:sns:eu-west-2:111111111111:mock-worker-events" + worker_topic_publish_policy_arn = "arn:aws:iam::111111111111:policy/mock-worker-topic-publish" + lambda_worker_queue_name = "mock-lambda-worker-queue" + lambda_worker_queue_arn = "arn:aws:sqs:eu-west-2:111111111111:mock-lambda-worker-queue" + lambda_worker_queue_url = "https://sqs.eu-west-2.amazonaws.com/111111111111/mock-lambda-worker-queue" + lambda_worker_queue_read_policy_arn = "arn:aws:iam::111111111111:policy/mock-lambda-worker-queue-read" + lambda_worker_dead_letter_queue_name = "mock-lambda-worker-dlq" + lambda_worker_dead_letter_queue_url = "https://sqs.eu-west-2.amazonaws.com/111111111111/mock-lambda-worker-dlq" + ecs_worker_queue_name = "mock-ecs-worker-queue" + ecs_worker_queue_url = "https://sqs.eu-west-2.amazonaws.com/111111111111/mock-ecs-worker-queue" + ecs_worker_queue_read_policy_arn = "arn:aws:iam::111111111111:policy/mock-ecs-worker-queue-read" + } + + mock_outputs_merge_strategy_with_state = "shallow" + mock_outputs_allowed_terraform_commands = ["validate", "plan", "destroy", "init", "show", "graph-dependencies", "output-module-groups"] +} + +dependency "database" { + config_path = "${get_original_terragrunt_dir()}/../database" + + mock_outputs = { + database_credentials_secret_arn = "arn:aws:secretsmanager:eu-west-2:111111111111:secret:mock-database-credentials" + database_readwrite_endpoint = "mock-database.cluster-abcdefghijkl.eu-west-2.rds.amazonaws.com" + database_name = "app" + database_port = 5432 + database_cluster_identifier = "mock-database-cluster" + } + + mock_outputs_merge_strategy_with_state = "shallow" + mock_outputs_allowed_terraform_commands = ["validate", "plan", "destroy", "init", "show", "graph-dependencies", "output-module-groups"] } terraform { source = "../../../../modules//aws//task_worker" } -inputs = merge(local.messaging.inputs, local.database.inputs) +inputs = merge( + dependency.messaging.outputs, + { + database_credentials_secret_arn = dependency.database.outputs.database_credentials_secret_arn + database_readwrite_endpoint = dependency.database.outputs.database_readwrite_endpoint + database_name = dependency.database.outputs.database_name + database_port = dependency.database.outputs.database_port + database_cluster_identifier = dependency.database.outputs.database_cluster_identifier + }, +) diff --git a/infra/root.hcl b/infra/root.hcl index d8a18c33..40099e1e 100644 --- a/infra/root.hcl +++ b/infra/root.hcl @@ -12,22 +12,15 @@ locals { global_vars = read_terragrunt_config(find_in_parent_folders("global_vars.hcl")) environment_vars = read_terragrunt_config(find_in_parent_folders("environment_vars.hcl")) - infra_root_dir = abspath(dirname(find_in_parent_folders("root.hcl"))) project_name = element(split("/", local.github_repo), 1) - aws_region = local.global_vars.inputs.aws_region - base_reference = "${local.aws_account_id}-${local.aws_region}-${local.project_name}" - deploy_role_name = "${local.project_name}-${local.environment}-github-oidc-role" - deploy_role_arn = "arn:aws:iam::${local.aws_account_id}:role/${local.deploy_role_name}" - state_bucket = "${local.base_reference}-tfstate" - plan_bucket = "${local.base_reference}-tfplan" - state_key = "${local.environment}/${local.provider}/${local.module}/terraform.tfstate" - plan_artifact_stack_key = "${local.environment}/${local.provider}/${local.module}" - plan_artifact_retention_days = try( - local.environment_vars.inputs.infra_plan_artifact_expiration_days, - 1, - ) + aws_region = local.global_vars.inputs.aws_region + base_reference = "${local.aws_account_id}-${local.aws_region}-${local.project_name}" + deploy_role_name = "${local.project_name}-${local.environment}-github-oidc-role" + deploy_role_arn = "arn:aws:iam::${local.aws_account_id}:role/${local.deploy_role_name}" + state_bucket = "${local.base_reference}-tfstate" + state_key = "${local.environment}/${local.provider}/${local.module}/terraform.tfstate" # separate shared artifact resources when dev, otherwise ci artifact_base = local.environment == "dev" ? "${local.base_reference}-${local.environment}" : "${local.base_reference}-ci" code_bucket = "${local.artifact_base}-code" @@ -41,41 +34,6 @@ terraform { "bash", "-c", "echo STATE:${local.state_bucket}/${local.state_key} LOCKFILE:${local.state_key}.tflock" ] } - - before_hook "ensure_plan_artifact_bucket" { - commands = ["init", "plan"] - execute = [ - "bash", - "${local.infra_root_dir}/scripts/ensure-plan-artifact-bucket.sh", - local.plan_bucket, - local.aws_region, - tostring(local.plan_artifact_retention_days), - ] - } - - before_hook "download_saved_plan" { - commands = ["apply"] - execute = [ - "bash", - "${local.infra_root_dir}/scripts/handle-plan-artifact.sh", - "download", - local.plan_artifact_stack_key, - local.plan_bucket, - local.environment, - ] - } - - after_hook "upload_saved_plan" { - commands = ["plan"] - execute = [ - "bash", - "${local.infra_root_dir}/scripts/handle-plan-artifact.sh", - "upload", - local.plan_artifact_stack_key, - local.plan_bucket, - local.environment, - ] - } } remote_state { @@ -142,7 +100,6 @@ inputs = merge( deploy_role_name = local.deploy_role_name deploy_role_arn = local.deploy_role_arn state_bucket = local.state_bucket - plan_bucket = local.plan_bucket code_bucket = local.code_bucket ecr_repository_name = local.ecr_repository_name } diff --git a/infra/scripts/ensure-plan-artifact-bucket.sh b/infra/scripts/ensure-plan-artifact-bucket.sh deleted file mode 100644 index b1596b3e..00000000 --- a/infra/scripts/ensure-plan-artifact-bucket.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -bucket_name="${1:?bucket name is required}" -aws_region="${2:?aws region is required}" -retention_days="${3:-0}" -plan_prefix="${INFRA_PLAN_DIR:-terragrunt_plan/}" -reset_flag="${TG_RESET_PLAN_ARTIFACT_BUCKET:-false}" - -if [[ "$plan_prefix" != */ ]]; then - plan_prefix="${plan_prefix}/" -fi - -ensure_lifecycle() { - if [[ "$retention_days" =~ ^[0-9]+$ ]] && [ "$retention_days" -gt 0 ]; then - aws s3api put-bucket-lifecycle-configuration \ - --bucket "$bucket_name" \ - --lifecycle-configuration "{ - \"Rules\": [ - { - \"ID\": \"expire-plan-artifacts\", - \"Status\": \"Enabled\", - \"Filter\": {\"Prefix\": \"$plan_prefix\"}, - \"Expiration\": {\"Days\": $retention_days} - } - ] - }" >/dev/null - echo "Ensured plan artifact retention of ${retention_days} days on s3://${bucket_name}/${plan_prefix}" - fi -} - -if aws s3api head-bucket --bucket "$bucket_name" >/dev/null 2>&1; then - if [ "$reset_flag" = "true" ]; then - ensure_lifecycle - fi - exit 0 -fi - -if [ -r /dev/tty ] && [ -w /dev/tty ]; then - printf "Plan bucket '%s' does not exist. Create it in %s? [y/N] " "$bucket_name" "$aws_region" > /dev/tty - read -r response < /dev/tty - case "$response" in - [yY]|[yY][eE][sS]) ;; - *) - echo "Plan bucket creation declined." >&2 - exit 1 - ;; - esac -else - echo "Plan bucket '$bucket_name' does not exist and no interactive terminal is available for confirmation." >&2 - echo "Create it manually or rerun from a terminal where Terragrunt hooks can prompt." >&2 - exit 1 -fi - -if [ "$aws_region" = "us-east-1" ]; then - aws s3api create-bucket --bucket "$bucket_name" >/dev/null -else - aws s3api create-bucket --bucket "$bucket_name" --create-bucket-configuration "LocationConstraint=$aws_region" >/dev/null -fi - -ensure_lifecycle diff --git a/infra/scripts/handle-plan-artifact.sh b/infra/scripts/handle-plan-artifact.sh deleted file mode 100644 index 4cee9200..00000000 --- a/infra/scripts/handle-plan-artifact.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -mode="${1:?mode is required}" -logical_tg_dir="${2:?terragrunt directory is required}" -plan_bucket="${3:?plan bucket is required}" -environment="${4:?environment is required}" -infra_plan_dir="${INFRA_PLAN_DIR:-terragrunt_plan}" - -plan_path="${PWD}/terragrunt.tfplan" -plan_text_path="${PWD}/terragrunt.plan.txt" -plan_meta_path="${PWD}/terragrunt.plan.meta.json" -plan_json_path="${PWD}/terragrunt.plan.json" -plan_log_path="${PWD}/${TG_PLAN_LOG_FILENAME:-terragrunt.plan.log}" -fallback_plan_log_path="${TG_PLAN_LOG_ABS_PATH:-}" - -if [[ "${TG_ENABLE_PLAN_ARTIFACTS:-false}" != "true" ]]; then - echo "TG_ENABLE_PLAN_ARTIFACTS=false, skipping plan artifact ${mode}." >&2 - exit 0 -fi - -if [[ -z "${PLAN_ARTIFACT_RUN_ID:-}" ]]; then - echo "PLAN_ARTIFACT_RUN_ID is required when TG_ENABLE_PLAN_ARTIFACTS=true." >&2 - exit 1 -fi - -sanitized_dir="$(echo "$logical_tg_dir" | tr '/.' '--')" -artifact_s3_prefix="s3://${plan_bucket}/${infra_plan_dir}/${environment}/${PLAN_ARTIFACT_RUN_ID}/terragrunt-plan-${sanitized_dir}" - -case "$mode" in - download) - echo "Downloading plan artifacts from ${artifact_s3_prefix}" >&2 - if ! aws s3 ls "${artifact_s3_prefix}/terragrunt.tfplan" >/dev/null 2>&1; then - echo "Saved plan artifact not found for ${logical_tg_dir} and PLAN_ARTIFACT_RUN_ID=${PLAN_ARTIFACT_RUN_ID}." >&2 - echo "Expected plan bundle at ${artifact_s3_prefix}" >&2 - exit 1 - fi - - aws s3 cp "${artifact_s3_prefix}/terragrunt.tfplan" "$plan_path" - aws s3 cp "${artifact_s3_prefix}/terragrunt.plan.txt" "$plan_text_path" - aws s3 cp "${artifact_s3_prefix}/terragrunt.plan.meta.json" "$plan_meta_path" - echo "Downloaded plan artifacts for ${logical_tg_dir}" >&2 - - if [[ "$(jq -r '.contains_mocked_outputs // false' "$plan_meta_path")" == "true" ]]; then - echo "Saved plan for '${logical_tg_dir}' contains mocked outputs. Regenerate it after upstream real outputs exist." >&2 - exit 1 - fi - ;; - upload) - if [[ ! -f "$plan_path" ]]; then - exit 0 - fi - - terraform show -no-color "$plan_path" > "$plan_text_path" - terraform show -json "$plan_path" > "$plan_json_path" - - contains_mocked_outputs=false - if [[ -f "$plan_log_path" ]] && grep -Fq "mock outputs provided and returning those in dependency output" "$plan_log_path"; then - contains_mocked_outputs=true - elif [[ -n "$fallback_plan_log_path" ]] && [[ -f "$fallback_plan_log_path" ]] && grep -Fq "mock outputs provided and returning those in dependency output" "$fallback_plan_log_path"; then - contains_mocked_outputs=true - fi - - jq -n \ - --arg tg_directory "$logical_tg_dir" \ - --argjson has_changes "$(jq -r '([(.resource_changes // [])[]?.change.actions[]?] | any(. != "no-op")) or ((.output_changes // {}) | length > 0)' "$plan_json_path")" \ - --argjson contains_mocked_outputs "$contains_mocked_outputs" \ - '{tg_directory: $tg_directory, has_changes: $has_changes, contains_mocked_outputs: $contains_mocked_outputs}' \ - > "$plan_meta_path" - - echo "Uploading plan artifacts for ${logical_tg_dir} to ${artifact_s3_prefix}" >&2 - aws s3 cp "$plan_path" "${artifact_s3_prefix}/terragrunt.tfplan" - aws s3 cp "$plan_text_path" "${artifact_s3_prefix}/terragrunt.plan.txt" - aws s3 cp "$plan_meta_path" "${artifact_s3_prefix}/terragrunt.plan.meta.json" - echo "Uploaded plan artifacts for ${logical_tg_dir}" >&2 - rm -f "$plan_json_path" - ;; - *) - echo "Unknown mode '$mode'." >&2 - exit 2 - ;; -esac diff --git a/justfile b/justfile index f29abaa9..c9af103b 100644 --- a/justfile +++ b/justfile @@ -152,6 +152,34 @@ tg-all op: terragrunt run-all {{op}} +# Print the raw Terragrunt run-all dependency graph. +tg-graph env provider='aws': + #!/usr/bin/env bash + set -euo pipefail + cd {{justfile_directory()}}/infra/live/{{env}}/{{provider}} + + terragrunt run-all graph-dependencies \ + --terragrunt-non-interactive \ + --terragrunt-include-external-dependencies \ + --terragrunt-log-level error + + +# Run tg-graph once locally and feed the raw output through the CI graph and +# wave processors. +tg-graph-waves env provider='aws': + #!/usr/bin/env bash + set -euo pipefail + cd {{justfile_directory()}} + + tg_graph_json="$( + TG_GRAPH_OUTPUT="$(just tg-graph "{{env}}" "{{provider}}")" \ + just --justfile "{{justfile_directory()}}/justfile.ci" tg-graph-output-to-json "{{env}}" "{{provider}}" + )" + + TG_GRAPH_JSON="$tg_graph_json" \ + just --justfile "{{justfile_directory()}}/justfile.ci" tg-graph-json-to-waves + + # Open an ECS Exec shell in the worker debug container. worker-debug-shell env: #!/usr/bin/env bash diff --git a/justfile.ci b/justfile.ci index c0ec334c..a2dd19c1 100644 --- a/justfile.ci +++ b/justfile.ci @@ -12,6 +12,130 @@ EXTRA_CONTAINER_DIRECTORIES := `just --justfile justfile --evaluate EXTRA_CONTAI NON_SERVICE_CONTAINER_DIRECTORIES := `just --justfile justfile --evaluate NON_SERVICE_CONTAINER_DIRECTORIES` +# Convert raw Terragrunt graph output from TG_GRAPH_OUTPUT into compact JSON. +tg-graph-output-to-json environment provider='aws': + #!/usr/bin/env bash + set -euo pipefail + cd "{{PROJECT_DIR}}" + + if [[ -z "${TG_GRAPH_OUTPUT:-}" ]]; then + echo "❌ TG_GRAPH_OUTPUT is required for tg-graph-output-to-json." + exit 1 + fi + + tmp_nodes="$(mktemp)" + tmp_edges="$(mktemp)" + trap 'rm -f "$tmp_nodes" "$tmp_edges"' EXIT + + awk -F'"' ' + /->/ && NF >= 4 { + from = $2 + to = $4 + sub(".*/", "", from) + sub(".*/", "", to) + print from "\t" to + next + } + /^[[:space:]]*"/ && /;[[:space:]]*$/ && NF >= 2 { + node = $2 + sub(".*/", "", node) + print node + } + ' <(printf '%s\n' "$TG_GRAPH_OUTPUT") \ + | while IFS= read -r line; do + if [[ "$line" == *$'\t'* ]]; then + printf '%s\n' "$line" >> "$tmp_edges" + elif [[ -n "$line" ]]; then + printf '%s\n' "$line" >> "$tmp_nodes" + fi + done + + { + cat "$tmp_nodes" + awk -F'\t' 'NF >= 2 { print $1; print $2 }' "$tmp_edges" + } \ + | sort -u \ + | while IFS= read -r node; do + [[ -n "$node" ]] || continue + deps="$( + awk -F'\t' -v target="$node" '$1 == target { print $2 }' "$tmp_edges" \ + | sort -u \ + | jq -R . \ + | jq -s -c . + )" + printf '%s\t%s\n' "$node" "$deps" + done \ + | jq -R -s --arg environment "{{environment}}" --arg provider "{{provider}}" ' + split("\n") + | map(select(length > 0)) + | map(split("\t")) + | map(select(length == 2)) + | { + environment: $environment, + provider: $provider, + dependencies: ( + reduce .[] as $pair + ({}; + .[$pair[0]] = ($pair[1] | fromjson) + ) + ) + } + ' \ + | jq -c ' + .dependencies as $deps + | . + { + nodes: ($deps | keys | sort), + edges: ( + $deps + | to_entries + | map(.key as $from | .value[]? | {from: $from, to: .}) + | sort_by(.from, .to) + ) + } + ' + + +# Convert compact Terragrunt graph JSON from TG_GRAPH_JSON into a dependency-safe +# wave matrix JSON array for GitHub Actions. +tg-graph-json-to-waves: + #!/usr/bin/env bash + set -euo pipefail + cd "{{PROJECT_DIR}}" + + if [[ -z "${TG_GRAPH_JSON:-}" ]]; then + echo "❌ TG_GRAPH_JSON is required for tg-graph-json-to-waves." + exit 1 + fi + + jq -cn --argjson graph "$TG_GRAPH_JSON" ' + def build_waves($remaining): + if ($remaining | length) == 0 then + [] + else + ($remaining | to_entries | map(select((.value | length) == 0) | .key) | sort) as $ready + | if ($ready | length) == 0 then + error("Cycle detected in Terragrunt dependency graph.") + else + [{modules: $ready}] + build_waves( + reduce ($remaining | to_entries[]) as $entry + ({}; + if ($ready | index($entry.key)) != null then + . + else + .[$entry.key] = ($entry.value - $ready) + end + ) + ) + end + end; + + ($graph.dependencies // {}) as $deps + | build_waves($deps) + | to_entries + | map({wave: .key, modules: .value.modules}) + ' | jq -c . + + # Run `tflint` across Terraform module directories. tf-lint-check: #!/bin/bash