diff --git a/.github/workflows/dns-zone-checks.yml b/.github/workflows/dns-zone-checks.yml deleted file mode 100644 index 8878e00d6..000000000 --- a/.github/workflows/dns-zone-checks.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: DNS Zone Checks - -# fmt/validate for the mapletestimony.org zone root. No plan: the zone's -# project has no CI planner, and no workflow applies terraform — applies are -# human-run (infra/gcp/dns/README.md). -on: - pull_request: - paths: - - infra/gcp/dns/** - - .github/workflows/dns-zone-checks.yml - - .github/actions/setup-terraform/** - push: - branches: - - main - paths: - - infra/gcp/dns/** - - .github/workflows/dns-zone-checks.yml - - .github/actions/setup-terraform/** - -env: - # Keep in lockstep with required_version in infra/gcp/dns/versions.tf. - TF_VERSION: 1.16.0 - -jobs: - fmt_and_validate: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/setup-terraform - with: - terraform_version: ${{ env.TF_VERSION }} - - name: Check formatting - run: terraform -chdir=infra/gcp/dns fmt -check -recursive - - name: Validate - run: | - terraform -chdir=infra/gcp/dns init -backend=false -input=false - terraform -chdir=infra/gcp/dns validate diff --git a/.github/workflows/terraform-checks.yml b/.github/workflows/terraform-checks.yml new file mode 100644 index 000000000..4bcb85404 --- /dev/null +++ b/.github/workflows/terraform-checks.yml @@ -0,0 +1,102 @@ +name: Terraform Checks + +# fmt/validate for every root in the matrix below on every PR; a dev plan (job +# summary) for infra/gcp on same-repo PRs. NO workflow applies terraform — +# applies are human-run (each root's README.md). +on: + pull_request: + paths: + - infra/gcp/** + - .github/workflows/terraform-checks.yml + - .github/actions/setup-terraform/** + push: + branches: + - main + paths: + - infra/gcp/** + - .github/workflows/terraform-checks.yml + - .github/actions/setup-terraform/** + +env: + # Keep in lockstep with required_version in each matrix root's versions.tf. + TF_VERSION: 1.16.0 + +jobs: + fmt_and_validate: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + root: + - infra/gcp + - infra/gcp/dns + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup-terraform + with: + terraform_version: ${{ env.TF_VERSION }} + - name: Check formatting + run: terraform -chdir=${{ matrix.root }} fmt -check -recursive + - name: Validate + run: | + terraform -chdir=${{ matrix.root }} init -backend=false -input=false + terraform -chdir=${{ matrix.root }} validate + + plan_dev: + needs: fmt_and_validate + # Upstream, same-repo PRs only: the terraform-plan environment holding the + # planner's GCP_SERVICE_ACCOUNT_KEY lives in codeforboston/maple, and fork + # PRs get no secrets either way. (Not `dev`: that environment's deployment + # branch policy allows only main, so a PR job bound to it can never start.) + if: github.repository_owner == 'codeforboston' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + environment: terraform-plan + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup-terraform + with: + terraform_version: ${{ env.TF_VERSION }} + # No wrapper: we redirect plan stdout to a file ourselves. + terraform_wrapper: "false" + - uses: google-github-actions/auth@v3 + with: + credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }} + - uses: google-github-actions/setup-gcloud@v3 + - name: Check for the state bucket + id: bootstrap + # Gate on the actual precondition rather than swallowing init errors: + # only a missing bucket (not yet bootstrapped, the runbook in + # infra/gcp/README.md) skips the plan. A 403 means the planner lacks + # its grant (ci_planner in envs/dev.tfvars, iam.tf) and is red, not + # "skipped" forever; so is any other failure. + run: | + bucket=$(sed -n 's/^bucket *= *"\(.*\)".*/\1/p' infra/gcp/envs/dev.gcs.tfbackend) + if err=$(gcloud storage ls "gs://$bucket" 2>&1 >/dev/null); then + echo "ok=true" >> "$GITHUB_OUTPUT" + elif grep -q "404\|NotFound\|not found" <<< "$err"; then + echo "ok=false" >> "$GITHUB_OUTPUT" + echo "state bucket gs://$bucket not bootstrapped yet (see infra/gcp/README.md); plan skipped" >> "$GITHUB_STEP_SUMMARY" + else + echo "cannot read gs://$bucket: $err" >&2 + exit 1 + fi + - name: Init + if: steps.bootstrap.outputs.ok == 'true' + run: terraform -chdir=infra/gcp init -backend-config=envs/dev.gcs.tfbackend -input=false + - name: Plan (dev) + if: steps.bootstrap.outputs.ok == 'true' + # -lock=false: an advisory CI plan must never leave a stale lock that + # blocks a human apply. -refresh=false: the plan is config-vs-state, so + # it needs no read access beyond the state bucket and the parent DNS + # zone (the data source in dns.tf), which are the planner's two grants. + run: | + terraform -chdir=infra/gcp plan -var-file=envs/dev.tfvars -input=false -no-color -lock=false -refresh=false > "$RUNNER_TEMP/plan.txt" + { + echo "### terraform plan — infra/gcp (dev)" + echo '
plan output' + echo + echo '```' + head -c 60000 "$RUNNER_TEMP/plan.txt" + echo '```' + echo '
' + } >> "$GITHUB_STEP_SUMMARY" diff --git a/docs/adr/0001-atproto-infra.md b/docs/adr/0001-atproto-infra.md new file mode 100644 index 000000000..48f7f441e --- /dev/null +++ b/docs/adr/0001-atproto-infra.md @@ -0,0 +1,49 @@ +# ADR 0001: atproto infrastructure on GCP + +- **Status:** Accepted +- **Date:** 2026-08-25 + +## Context + +atproto phase 1 puts MAPLE's legislative data on a PDS: a long-lived, stateful service +(SQLite on local disk, plus a blobstore). MAPLE's application state and backend compute +live in the Firebase projects `digital-testimony-dev` and `digital-testimony-prod`. The +only other infrastructure is Typesense on AWS, managed from a repo this team cannot write +to, deployed to dev and prod together with no gate, unchanged since June 2023. + +We hold `roles/editor` on dev and nothing on prod; prod waits on a handoff to the upstream +maintainers. + +## Decision + +New atproto infrastructure runs in the existing GCP projects, one environment per project, +defined as Terraform in this repo under `infra/gcp`. Terraform owns only the atproto +resources; Firestore, Cloud Functions and Firebase extensions stay with the `firebase` +CLI. Secret values are never Terraform inputs: state stores them in plaintext, so they are +added out of band. + +The PDS is a Compute Engine VM running the reference `bluesky-social/pds` image, its data +on a persistent disk, blobs in a Cloud Storage bucket. The PDS expects a POSIX filesystem +for SQLite, and Bluesky's installer targets exactly this shape. + +## Consequences + +- Production is a reviewable `terraform apply` run by an owner, not a request for owner + access for an outside contributor. +- Dev and prod are the same module with different variables. +- We patch a VM. A managed PDS would absorb that at the cost of custody of the data and + keys. +- Terraform is a second IaC tool beside the AWS CDK; that stack is AWS-only and + unmaintained. + +## Alternatives + +- **The AWS ECS cluster.** Not writable by this team, no dev/prod gate, non-durable + storage, unmaintained. +- **Cloud Run for the PDS.** No block storage; SQLite over GCS FUSE or Filestore is + unsafe. +- **GKE.** More moving parts than one small service justifies. +- **A managed PDS.** Gives up custody of data and signing keys for a project whose point + is custody of public records. +- **Hand-run `gcloud` commands.** Makes production an access request rather than a + reviewable plan, and drifts from what is deployed. diff --git a/infra/gcp/.gitignore b/infra/gcp/.gitignore new file mode 100644 index 000000000..65ebd7b69 --- /dev/null +++ b/infra/gcp/.gitignore @@ -0,0 +1 @@ +**/.terraform/ diff --git a/infra/gcp/.terraform.lock.hcl b/infra/gcp/.terraform.lock.hcl new file mode 100644 index 000000000..a1ee03c77 --- /dev/null +++ b/infra/gcp/.terraform.lock.hcl @@ -0,0 +1,24 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/google" { + version = "7.46.0" + constraints = "~> 7.46" + hashes = [ + "h1:L3O+5UStRwh0MTseTQ7/IlOruySgkWvZJuLMrCs4/D4=", + "h1:RW6+H+fkavdUf/iB0t951fjOB6ix3tUgkozRHjksKOc=", + "h1:lf5PDcsxzVgO5ZHpl4WRr47Ft+HCxq9v9lbdUjm3qag=", + "zh:0d470c9e58a9f228a337cd52d7137943ac169511c13b4a8f3ecb075247b608d8", + "zh:28a14d1a8b1de00af2a527b92eb3695b5e5665b50a6ada8ba7c0367b63b4676b", + "zh:39cb9cea9ffff16178eec67cefccbcb80401dd38dce324d37209e9005f16d21d", + "zh:5637a959ed5413c8c4e656ba8cf266b8ba40193b10e9d8a54e558caf3225191e", + "zh:626dfd817b2d02375f3152a89ffa31ef02f3e907820bf534e8aaad19245b3058", + "zh:6acad5498724d6b40156ced010b6fe949b85d421e655df46c3e430f410f927e0", + "zh:ba06957cd47a61bfd662e02bb6e8760f1674436d5e3ca88b6b0e1000cc4d7f37", + "zh:dd24a83416deb21e56b70d3bf2e5e99af414868d4d686ab4a85284c2d90aacb2", + "zh:eb99197f62f84ea5ac29a607b5046e239427045908952d926ec5d9186f60f75e", + "zh:f4aa9fb2af06d8c81fc42a2db5c6223186832cedbae8d791ea5b811867dad019", + "zh:f4fb1431737107838ab48cff4db6eb711c6d5bf7ca64100ec9a85d2108e0265f", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} diff --git a/infra/gcp/CI.md b/infra/gcp/CI.md new file mode 100644 index 000000000..0b7026497 --- /dev/null +++ b/infra/gcp/CI.md @@ -0,0 +1,40 @@ +# Terraform CI + +`.github/workflows/terraform-checks.yml` runs `fmt -check` and `validate` on both roots (this one +and `dns/`) for every PR, with no setup. Its `plan_dev` job also posts an advisory dev plan to the +job summary, on same-repo PRs in codeforboston/maple only (fork PRs get no secrets and show it +skipped; the comment on that job has the details). Turning the plan on is one-time work for an +owner of `digital-testimony-dev` with admin on codeforboston/maple: + +1. **The planner identity.** A service account with no roles of its own: + + ```sh + gcloud iam service-accounts create atproto-ci-planner --project=digital-testimony-dev \ + --display-name="terraform plan from GitHub Actions" + ``` + +2. **Its two grants.** In `envs/dev.tfvars` set + `ci_planner = "serviceAccount:atproto-ci-planner@digital-testimony-dev.iam.gserviceaccount.com"`, + merge it like any other change, and apply as an owner of both projects (see Permissions; the + grants are the `ci_planner_*` members in `iam.tf`: read-only on the state bucket and on the + parent DNS zone's project). +3. **A key.** The one step outside Terraform, so the key material never enters state. Outside the + working tree too — a service-account private key sitting in the repo is one `git add -A` away + from being committed: + + ```sh + keydir=$(mktemp -d) + gcloud iam service-accounts keys create "$keydir/planner.json" \ + --iam-account=atproto-ci-planner@digital-testimony-dev.iam.gserviceaccount.com + ``` + +4. **The GitHub environment.** codeforboston/maple → Settings → Environments → New environment, + name `terraform-plan`, deployment branches and tags **No restriction**, no required reviewers. + Environment secrets → Add → `GCP_SERVICE_ACCOUNT_KEY`, value: the contents of + `$keydir/planner.json`. Then `rm -rf "$keydir"`. +5. **Check.** A PR from a branch in codeforboston/maple that touches `infra/gcp/**` shows the plan + in the `plan_dev` job summary. A 404 in "Check for the state bucket" means + `infra/gcp/scripts/bootstrap.sh dev` has not run; a 403 means step 2 has not landed. + +Rotate or revoke with `gcloud iam service-accounts keys list|delete --iam-account=…`, then redo +step 4. diff --git a/infra/gcp/README.md b/infra/gcp/README.md new file mode 100644 index 000000000..466f87f93 --- /dev/null +++ b/infra/gcp/README.md @@ -0,0 +1,68 @@ +# atproto PDS on GCP + +One Terraform root, one state per environment (`envs/.*`). Applies are human-run; CI only +plans. Design: [ADR 0001](../../docs/adr/0001-atproto-infra.md). + +## Permissions + +- **The environment's project**: `roles/editor` for every step, plus `storage.hmacKeys.create` for + step 3. `roles/owner` for the grants in `iam.tf`: an editor's apply ends red on the grants only, + and an owner's apply afterwards plans exactly those. +- **`digital-testimony-prod`**: `roles/dns.admin` for the NS record in the parent zone. Every plan + reads that zone, so without at least `roles/dns.reader` nothing plans. + +## Apply + +`infra/gcp/dns` is applied first (this root looks its zone up). The hostname is apply-once +(it lands in the DID document of every account the PDS creates). + +```sh +infra/gcp/scripts/bootstrap.sh dev # 1. APIs and the state bucket +terraform -chdir=infra/gcp init -backend-config=envs/dev.gcs.tfbackend +terraform -chdir=infra/gcp apply -var-file=envs/dev.tfvars # 2. everything below +infra/gcp/scripts/secrets.sh dev # 3. the five secret versions; never rotates +curl https://pds-dev.mapletestimony.org/xrpc/_health # the VM starts the PDS within 3 min +gcloud storage ls gs://digital-testimony-dev-atproto-pds-blobs/ # done once one uploadBlob lands here +gcloud compute instances get-serial-port-output atproto-pds --zone=us-central1-a | grep pds-startup # if not +``` + +Prod: the same with `prod`. + +## What gets applied + +A static IP and firewall (80/443 open, 22 via IAP); the `atproto-pds` VM with a 20 GB data disk +snapshotted daily for 14 days; the delegated zone, its A record and the parent NS record; five +Secret Manager secrets without versions; the blob bucket; the VM's service account and its +grants. Knobs: `envs/.tfvars`. Not here: secret versions and the HMAC key (`secrets.sh`), +the state bucket (`bootstrap.sh`). + +## Rollback + +- **Config**: revert and apply. A startup-script change lands on the next boot: + `gcloud compute instances reset atproto-pds --zone=us-central1-a` applies it now. +- **A secret**: `gcloud secrets versions add --data-file=-`, reset the VM (it re-reads every + secret on boot, nothing on disk), disable the old version. +- **Data**: create a disk from a snapshot and attach it as `pds-data`. Blobs are in the bucket. +- **State**: the bucket is versioned; restore the earlier object. +- **Teardown**: `destroy` refuses by design (`prevent_destroy` on disk, bucket and zone; the VM + is deletion-protected). Lifting those is its own reviewed change. + +## Monitoring + +Three alerts, all to `alert_channels` in `envs/.tfvars` (dev: one email; prod: the pager, a +channel-type swap there changes no policy). Subjects start with `[]`, and each page carries +its own first step; this section is what a page cannot. + +- **PDS down** (critical): `https:///xrpc/_health` failing from two regions for 5 min. + One check covers VM, docker, caddy, cert expiry and DNS, from where the relay stands. Expect one + during bring-up: that page is the channel test. +- **Disk ≥ 80%** on any of the VM's disks, and **memory ≥ 90%** for 10 min, via the Ops Agent the + startup script installs (`pds-startup.sh.tftpl`). + +Prove it once per environment: `gcloud compute ssh atproto-pds --tunnel-through-iap --zone=us-central1-a`, +`sudo systemctl stop pds.service`, wait for the page (≤ 6 min), `start` it. + +## CI + +`.github/workflows/terraform-checks.yml`: `fmt`, `validate` and an advisory dev plan on PRs. What +runs, and the one-time setup the plan needs: [CI.md](CI.md). diff --git a/infra/gcp/accounts.tf b/infra/gcp/accounts.tf new file mode 100644 index 000000000..36015dd8c --- /dev/null +++ b/infra/gcp/accounts.tf @@ -0,0 +1,4 @@ +resource "google_service_account" "pds" { + account_id = "atproto-pds" + display_name = "atproto PDS VM" +} diff --git a/infra/gcp/backend.tf b/infra/gcp/backend.tf new file mode 100644 index 000000000..a75676da5 --- /dev/null +++ b/infra/gcp/backend.tf @@ -0,0 +1,6 @@ +# Partial backend configuration — the bucket/prefix come from +# envs/.gcs.tfbackend at init time: +# terraform init -backend-config=envs/dev.gcs.tfbackend +terraform { + backend "gcs" {} +} diff --git a/infra/gcp/blobs.tf b/infra/gcp/blobs.tf new file mode 100644 index 000000000..c17293a95 --- /dev/null +++ b/infra/gcp/blobs.tf @@ -0,0 +1,26 @@ +# Blob storage for the PDS: uploaded media, content-addressed by CID. The PDS +# talks to it through its S3 blobstore over Cloud Storage's S3-compatible XML +# API (storage.googleapis.com, HMAC credentials), so the data disk holds only +# SQLite and the actor store and never grows with uploads. +# +# The HMAC key is NOT a Terraform resource: google_storage_hmac_key stores the +# secret in state (ADR 0001). README.md step 3 creates it +# with gcloud for the PDS service account and adds both halves to Secret +# Manager; the key inherits the account's IAM, granted in iam.tf. +resource "google_storage_bucket" "pds_blobs" { + name = local.blob_bucket + location = var.region + labels = local.labels + + uniform_bucket_level_access = true + public_access_prevention = "enforced" + + # Blobs are immutable and content-addressed; object versioning would only + # double the bill. The default 7-day soft delete covers accidental deletes. + + # Deliberate destruction means removing this block first, in its own + # reviewed change — same rule as the data disk. + lifecycle { + prevent_destroy = true + } +} diff --git a/infra/gcp/dns.tf b/infra/gcp/dns.tf new file mode 100644 index 000000000..9e866fe0c --- /dev/null +++ b/infra/gcp/dns.tf @@ -0,0 +1,59 @@ +# Delegated zone for the PDS. The parent mapletestimony.org zone is the Cloud +# DNS zone the infra/gcp/dns root owns in digital-testimony-prod; the +# delegation is ONE NS record there, pointing at this zone's name servers +# (parent_ns below). Until the parent zone is authoritative +# (infra/gcp/dns/README.md) this zone is inert: nothing +# resolves, Caddy's HTTP-01 challenge cannot complete, and the PDS is +# unreachable over TLS. +resource "google_dns_managed_zone" "pds" { + name = "atproto-pds" + dns_name = "${var.pds_hostname}." + description = "Delegated zone for the atproto PDS (parent NS record: google_dns_record_set.parent_ns)" + + # dns_name forces replacement, and pds_hostname is apply-once (it is in the + # DID document). A recreated zone gets new name servers, orphaning the + # by-hand parent NS record. Same rule as the data disk: destroying this is + # its own reviewed change. + lifecycle { + prevent_destroy = true + } +} + +resource "google_dns_record_set" "pds_a" { + managed_zone = google_dns_managed_zone.pds.name + name = google_dns_managed_zone.pds.dns_name + type = "A" + ttl = 300 + rrdatas = [google_compute_address.pds.address] +} + +# The parent zone, looked up rather than copied: infra/gcp/dns owns it, in the +# prod project for every environment, and applies first. A plan fails here if +# it is missing. Reading it needs dns.reader in that project (the CI planner's +# grant in iam.tf); writing the record below needs dns.admin. The other +# project is reached through `project` alone; no provider alias. +locals { + parent_zone = { project = "digital-testimony-prod", name = "mapletestimony-org" } +} + +data "google_dns_managed_zone" "parent" { + project = local.parent_zone.project + name = local.parent_zone.name +} + +# The delegation, in the parent zone. +resource "google_dns_record_set" "parent_ns" { + project = data.google_dns_managed_zone.parent.project + managed_zone = data.google_dns_managed_zone.parent.name + name = google_dns_managed_zone.pds.dns_name + type = "NS" + ttl = 3600 # delegations are stable; 1h keeps resolver churn down + rrdatas = google_dns_managed_zone.pds.name_servers + + lifecycle { + precondition { + condition = endswith(google_dns_managed_zone.pds.dns_name, ".${data.google_dns_managed_zone.parent.dns_name}") + error_message = "pds_hostname must be a subdomain of the parent zone ${data.google_dns_managed_zone.parent.dns_name}" + } + } +} diff --git a/infra/gcp/dns/versions.tf b/infra/gcp/dns/versions.tf index 60f9af2ac..f9753a0b2 100644 --- a/infra/gcp/dns/versions.tf +++ b/infra/gcp/dns/versions.tf @@ -1,4 +1,4 @@ -# Keep terraform_version in .github/workflows/dns-zone-checks.yml in lockstep +# Keep terraform_version in .github/workflows/terraform-checks.yml in lockstep # with required_version here. terraform { required_version = "~> 1.16.0" diff --git a/infra/gcp/envs/dev.gcs.tfbackend b/infra/gcp/envs/dev.gcs.tfbackend new file mode 100644 index 000000000..707f09499 --- /dev/null +++ b/infra/gcp/envs/dev.gcs.tfbackend @@ -0,0 +1,4 @@ +# Created by the bootstrap runbook in ../README.md. Bucket names are global — +# if this one is taken, this file is the one place the name lives. +bucket = "digital-testimony-dev-tfstate" +prefix = "atproto" diff --git a/infra/gcp/envs/dev.tfvars b/infra/gcp/envs/dev.tfvars new file mode 100644 index 000000000..1b24a9467 --- /dev/null +++ b/infra/gcp/envs/dev.tfvars @@ -0,0 +1,23 @@ +env = "dev" +project_id = "digital-testimony-dev" + +# APPLY-ONCE: the hostname is baked into the DID document at account creation +# and is effectively immutable. Environments get sibling names under the +# parent domain — pds-dev here, pds for prod — so neither zone nests inside the +# other and the clean name stays reserved for production. +pds_hostname = "pds-dev.mapletestimony.org" + +# Dev is announced to the public relay so what it emits is visible off the +# box. Set to "" to run dark. +pds_crawlers = "https://bsky.network" + +# The service account behind the terraform-plan GitHub environment's +# GCP_SERVICE_ACCOUNT_KEY (CI.md). Read-only on the state bucket. Set it +# once that environment exists in codeforboston/maple; null = plans skipped. +ci_planner = null + +# Where dev's alerts go: one person, by email. Prod's pager is prod's own +# entry, never this one (variables.tf). +alert_channels = { + email = { type = "email", labels = { email_address = "violet@hypha.coop" } } +} diff --git a/infra/gcp/envs/prod.gcs.tfbackend b/infra/gcp/envs/prod.gcs.tfbackend new file mode 100644 index 000000000..d5f488e59 --- /dev/null +++ b/infra/gcp/envs/prod.gcs.tfbackend @@ -0,0 +1,4 @@ +# Prod state lives in the prod project (ADR 0001). Bucket is created by an +# owner at prod handoff — see ../README.md. +bucket = "digital-testimony-prod-tfstate" +prefix = "atproto" diff --git a/infra/gcp/envs/prod.tfvars b/infra/gcp/envs/prod.tfvars new file mode 100644 index 000000000..8471dbe71 --- /dev/null +++ b/infra/gcp/envs/prod.tfvars @@ -0,0 +1,22 @@ +# digital-testimony-prod is inaccessible until the prod handoff; this +# file is applied then, by an owner, and confirmed with the upstream owners +# first. Only the hostname is decided already. +env = "prod" +project_id = "digital-testimony-prod" + +# APPLY-ONCE. Reserved for production on purpose — dev lives at the sibling +# pds-dev.mapletestimony.org so the clean name is never burned into a dev DID. +# Confirm with the identity and handoff owners before the first prod apply. +pds_hostname = "pds.mapletestimony.org" + +# The public relay; without it MAPLE's records reach nobody. +pds_crawlers = "https://bsky.network" + +# No CI plans against prod: applies there are rare, human, and reviewed live. +ci_planner = null + +# Where prod's alerts go: the pager, decided at handoff. Deliberately unset +# (no default): a prod plan refuses to run until someone is on the other end. +# Email, or a pager's email-integration address, needs nothing else: +# alert_channels = { pager = { type = "email", labels = { email_address = "..." } } } +# A pager that needs a token: see alert_channels in variables.tf first. diff --git a/infra/gcp/iam.tf b/infra/gcp/iam.tf new file mode 100644 index 000000000..e6bc3440b --- /dev/null +++ b/infra/gcp/iam.tf @@ -0,0 +1,63 @@ +# Every IAM grant for the atproto stack, in the same root as the resources it +# grants on. Two rules: +# +# - The *_iam_member form only, NEVER *_iam_binding: binding is authoritative +# for its role and would silently strip the other owners of these shared +# projects. +# - Editors can PLAN this file but not APPLY it: roles/editor lacks every +# setIamPolicy. Nothing else here depends on a grant, so an editor's apply +# creates everything else and fails only these; an owner's apply afterwards +# plans exactly the grants and nothing more, which is the review artifact +# ADR 0001 asks for. Without the grants the PDS does not run: its +# service account cannot read its secrets, write its blobs, or log. + +# Secret-level (not project-level): the PDS VM reads exactly its five secrets +# at boot and nothing else. +resource "google_secret_manager_secret_iam_member" "pds_secret_access" { + for_each = google_secret_manager_secret.pds + secret_id = each.value.secret_id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${google_service_account.pds.email}" +} + +# Bucket-level: the PDS's HMAC key inherits this and reaches this one bucket +# only. objectAdmin because the PDS deletes blobs when a record that referenced +# them is deleted. +resource "google_storage_bucket_iam_member" "pds_blobs_object_admin" { + bucket = google_storage_bucket.pds_blobs.name + role = "roles/storage.objectAdmin" + member = "serviceAccount:${google_service_account.pds.email}" +} + +resource "google_project_iam_member" "pds_log_writer" { + project = var.project_id + role = "roles/logging.logWriter" + member = "serviceAccount:${google_service_account.pds.email}" +} + +resource "google_project_iam_member" "pds_metric_writer" { + project = var.project_id + role = "roles/monitoring.metricWriter" + member = "serviceAccount:${google_service_account.pds.email}" +} + +# CI plans (terraform-checks.yml) read this environment's state and the parent +# DNS zone, nothing else: `plan -refresh=false -lock=false` needs the state +# object, the workflow's bootstrap gate lists the bucket, and the data source +# in dns.tf reads the zone. Neither grant can write state or touch a resource, +# so a leaked CI key cannot apply. +resource "google_storage_bucket_iam_member" "ci_planner_state_reader" { + count = var.ci_planner == null ? 0 : 1 + bucket = local.state_bucket + role = "roles/storage.objectViewer" + member = var.ci_planner +} + +# Applying this one needs an owner of the PARENT zone's project, like the NS +# record itself (dns.tf). +resource "google_project_iam_member" "ci_planner_parent_zone_reader" { + count = var.ci_planner == null ? 0 : 1 + project = local.parent_zone.project + role = "roles/dns.reader" + member = var.ci_planner +} diff --git a/infra/gcp/locals.tf b/infra/gcp/locals.tf new file mode 100644 index 000000000..6d8a8334a --- /dev/null +++ b/infra/gcp/locals.tf @@ -0,0 +1,38 @@ +locals { + # Pinned here so this root applies standalone. The local harness + # (infra/atproto/images.env, on the atproto-core branch) pins the same PDS + # tag; bump both together once that branch lands. Caddy fronts only the + # deployed VM and has no harness counterpart. + pds_image = "ghcr.io/bluesky-social/pds:0.4.5027" + caddy_image = "caddy:2.10" + + # The state bucket is created by scripts/bootstrap.sh, not managed here; its + # name lives in envs/.gcs.tfbackend and nowhere else, so read it from + # there for the CI planner's grant (iam.tf) rather than typing it twice. + # (?m)^ so a commented-out `bucket = "…"` line cannot win: the CI workflow + # and bootstrap.sh both read this file with an anchored sed, and a grant on + # the wrong bucket would be silent. + state_bucket = regex("(?m)^bucket\\s*=\\s*\"([^\"]+)\"", file("${path.module}/envs/${var.env}.gcs.tfbackend"))[0] + + # PDS env var -> Secret Manager secret id. The one definition in this root: + # drives the secret resources (secrets.tf), their accessor grants (iam.tf), + # the boot-time fetch loop and the pds.env lines + # (templates/pds-startup.sh.tftpl), and the pds_secrets output. + pds_secrets = { + PDS_ADMIN_PASSWORD = "atproto-pds-admin-password" + PDS_JWT_SECRET = "atproto-pds-jwt-secret" + PDS_PLC_ROTATION_KEY_K256_PRIVATE_KEY_HEX = "atproto-pds-plc-rotation-key" + # The HMAC key pair for the blob bucket (blobs.tf). Created out of band + # with `gcloud storage hmac create` for the PDS service account: the + # google_storage_hmac_key resource would put the secret in state. + PDS_BLOBSTORE_S3_ACCESS_KEY_ID = "atproto-pds-blob-access-key-id" + PDS_BLOBSTORE_S3_SECRET_ACCESS_KEY = "atproto-pds-blob-secret-access-key" + } + + blob_bucket = "${var.project_id}-atproto-pds-blobs" + + labels = { + app = "atproto" + env = var.env + } +} diff --git a/infra/gcp/monitoring-pds.tf b/infra/gcp/monitoring-pds.tf new file mode 100644 index 000000000..a933bc7ce --- /dev/null +++ b/infra/gcp/monitoring-pds.tf @@ -0,0 +1,157 @@ +# What we watch on the PDS: unreachable from outside, disk full, memory gone. +# Everything notifies local.alert_channel_ids (monitoring.tf). + +# --- Reachability ------------------------------------------------------------ +# One HTTPS check from Google's checkers covers the VM, docker, caddy, the +# certificate and the DNS delegation at once, from the same vantage point as +# the relay: relay acceptance dies silently when TLS does, and this is what +# notices. The Caddyfile proxies the whole hostname to the PDS, so the +# endpoint is public. Expect one page during bring-up (delegation, +# certificate, the 3-minute PDS start); it doubles as the proof that the +# channel works. +resource "google_monitoring_uptime_check_config" "pds_health" { + display_name = "${google_compute_instance.pds.name} ${var.env} /xrpc/_health" + period = "60s" + timeout = "10s" + + http_check { + path = "/xrpc/_health" + port = 443 + use_ssl = true + # An expired certificate pages before the relay quietly drops us. + validate_ssl = true + } + + monitored_resource { + type = "uptime_url" + labels = { + project_id = var.project_id + host = var.pds_hostname + } + } + + user_labels = local.labels +} + +resource "google_monitoring_alert_policy" "pds_down" { + display_name = "${google_compute_instance.pds.name} ${var.env}: down" + combiner = "OR" + severity = "CRITICAL" + notification_channels = local.alert_channel_ids + user_labels = local.labels + + conditions { + display_name = "/xrpc/_health failing from two or more regions for 5 minutes" + + # The console's own recipe for an uptime check: over each 20-minute window + # count the checkers whose latest result is false; more than one of them, + # sustained 5 minutes, is an outage rather than one checker's bad route. + condition_threshold { + filter = "metric.type = \"monitoring.googleapis.com/uptime_check/check_passed\" AND metric.labels.check_id = \"${google_monitoring_uptime_check_config.pds_health.uptime_check_id}\" AND resource.type = \"uptime_url\"" + comparison = "COMPARISON_GT" + threshold_value = 1 + duration = "300s" + + aggregations { + alignment_period = "1200s" + per_series_aligner = "ALIGN_NEXT_OLDER" + cross_series_reducer = "REDUCE_COUNT_FALSE" + group_by_fields = ["resource.label.*"] + } + } + } + + documentation { + mime_type = "text/markdown" + subject = "[${var.env}] PDS down: https://${var.pds_hostname}/xrpc/_health" + content = <<-EOT + `https://${var.pds_hostname}/xrpc/_health` has failed from two or more regions for 5 minutes: the VM, docker, caddy, the certificate or the DNS delegation. First look: + + gcloud compute instances get-serial-port-output ${google_compute_instance.pds.name} --project=${var.project_id} --zone=${var.zone} | grep pds-startup + + ${local.alert_runbook} + EOT + } +} + +# --- Disk and memory (Ops Agent) --------------------------------------------- +# The default GCE metric set has neither filesystem usage nor memory; the Ops +# Agent (templates/pds-startup.sh.tftpl) adds both under agent.googleapis.com. +locals { + pds_instance_filter = "resource.type = \"gce_instance\" AND resource.labels.instance_id = \"${google_compute_instance.pds.instance_id}\"" +} + +# Every real disk: /pds full is the PDS's characteristic death (SQLite actor +# store), and the 10 GB boot disk holds docker's images and logs. The metric's +# device label is the kernel name without /dev, and the agent reports tmpfs, +# udev and docker's overlay mounts under it too; the pattern admits the block +# device prefixes GCE uses (scsi, nvme, virtio, xen) and nothing else, so a +# machine family that enumerates disks differently stays covered. +resource "google_monitoring_alert_policy" "pds_disk" { + display_name = "${google_compute_instance.pds.name} ${var.env}: disk over 80%" + combiner = "OR" + severity = "WARNING" + notification_channels = local.alert_channel_ids + user_labels = local.labels + + conditions { + display_name = "a disk on the PDS VM is over 80% used for 5 minutes" + + condition_threshold { + filter = "metric.type = \"agent.googleapis.com/disk/percent_used\" AND metric.labels.state = \"used\" AND metric.labels.device = monitoring.regex.full_match(\"(sd|nvme|vd|xvd)[a-z0-9]+\") AND ${local.pds_instance_filter}" + comparison = "COMPARISON_GT" + threshold_value = 80 + duration = "300s" + + aggregations { + alignment_period = "60s" + per_series_aligner = "ALIGN_MEAN" + group_by_fields = ["metric.label.device"] + } + } + } + + documentation { + mime_type = "text/markdown" + subject = "[${var.env}] PDS disk over 80%" + content = <<-EOT + The incident names the device: the second disk (sdb on this machine family) is `/pds`, the PDS's data, and full it stops the PDS; the first is the boot disk with docker. Grow `/pds` with `data_disk_size_gb` in `envs/${var.env}.tfvars`, apply, then `resize2fs` on the box. + + ${local.alert_runbook} + EOT + } +} + +resource "google_monitoring_alert_policy" "pds_memory" { + display_name = "${google_compute_instance.pds.name} ${var.env}: memory over 90%" + combiner = "OR" + severity = "WARNING" + notification_channels = local.alert_channel_ids + user_labels = local.labels + + conditions { + display_name = "memory on the PDS VM is over 90% used for 10 minutes" + + condition_threshold { + filter = "metric.type = \"agent.googleapis.com/memory/percent_used\" AND metric.labels.state = \"used\" AND ${local.pds_instance_filter}" + comparison = "COMPARISON_GT" + threshold_value = 90 + duration = "600s" + + aggregations { + alignment_period = "60s" + per_series_aligner = "ALIGN_MEAN" + } + } + } + + documentation { + mime_type = "text/markdown" + subject = "[${var.env}] PDS memory over 90%" + content = <<-EOT + The PDS, caddy and the Ops Agent share the VM's memory (${var.machine_type}), and the next step is the OOM killer taking the PDS. `machine_type` in `envs/${var.env}.tfvars` is the knob. + + ${local.alert_runbook} + EOT + } +} diff --git a/infra/gcp/monitoring.tf b/infra/gcp/monitoring.tf new file mode 100644 index 000000000..851b5a16f --- /dev/null +++ b/infra/gcp/monitoring.tf @@ -0,0 +1,18 @@ +# Where alerts go, shared by every alert policy in this root; per-component +# policies live in monitoring-.tf against the locals below. +# +# One channel per entry in alert_channels (envs/.tfvars). Swapping dev's +# email for prod's pager is a tfvars change and nothing else. +resource "google_monitoring_notification_channel" "alert" { + for_each = var.alert_channels + + display_name = "atproto ${var.env} ${each.key}" + type = each.value.type + labels = each.value.labels + user_labels = local.labels +} + +locals { + alert_channel_ids = [for c in google_monitoring_notification_channel.alert : c.id] + alert_runbook = "Runbook: infra/gcp/README.md, Monitoring." +} diff --git a/infra/gcp/network.tf b/infra/gcp/network.tf new file mode 100644 index 000000000..24e89050f --- /dev/null +++ b/infra/gcp/network.tf @@ -0,0 +1,30 @@ +resource "google_compute_address" "pds" { + name = "atproto-pds" + description = "Static external IP for the PDS; the A record in dns.tf points here." +} + +resource "google_compute_firewall" "pds_web" { + name = "atproto-pds-allow-web" + network = var.network + target_tags = ["atproto-pds"] + source_ranges = ["0.0.0.0/0"] + + allow { + protocol = "tcp" + ports = ["80", "443"] + } +} + +# SSH only via IAP TCP forwarding (gcloud compute ssh --tunnel-through-iap); +# no public port 22. +resource "google_compute_firewall" "pds_ssh_iap" { + name = "atproto-pds-allow-ssh-iap" + network = var.network + target_tags = ["atproto-pds"] + source_ranges = ["35.235.240.0/20"] + + allow { + protocol = "tcp" + ports = ["22"] + } +} diff --git a/infra/gcp/outputs.tf b/infra/gcp/outputs.tf new file mode 100644 index 000000000..01b61ec3b --- /dev/null +++ b/infra/gcp/outputs.tf @@ -0,0 +1,32 @@ +output "pds_zone_name_servers" { + description = "Name servers of the delegated PDS zone, as written into the parent zone's NS record (dns.tf). Read back here to check the delegation." + value = google_dns_managed_zone.pds.name_servers +} + +output "pds_static_ip" { + description = "Static external IP of the PDS VM." + value = google_compute_address.pds.address +} + +output "pds_blob_bucket" { + description = "GCS bucket the PDS stores blobs in; the HMAC key for it is created out of band (README step 3)." + value = google_storage_bucket.pds_blobs.name +} + +output "pds_service_account" { + description = "The PDS VM's service account; secrets.sh mints the blob HMAC key for it." + value = google_service_account.pds.email +} + +output "pds_instance" { + description = "The PDS VM, for secrets.sh's serial-console hint and for `gcloud compute ssh --tunnel-through-iap`." + value = { + name = google_compute_instance.pds.name + zone = google_compute_instance.pds.zone + } +} + +output "pds_secrets" { + description = "PDS env var -> Secret Manager secret id (locals.tf). secrets.sh reads this so the ids are typed in exactly one place." + value = local.pds_secrets +} diff --git a/infra/gcp/pds.tf b/infra/gcp/pds.tf new file mode 100644 index 000000000..9b181475e --- /dev/null +++ b/infra/gcp/pds.tf @@ -0,0 +1,98 @@ +resource "google_compute_disk" "pds_data" { + name = "atproto-pds-data" + type = "pd-balanced" + zone = var.zone + size = var.data_disk_size_gb + labels = local.labels + + # The PDS's SQLite databases and actor store live here (blobs: blobs.tf). + # Deliberate destruction + # means removing this block first, in its own reviewed change. + lifecycle { + prevent_destroy = true + } +} + +resource "google_compute_resource_policy" "pds_snapshots" { + name = "atproto-pds-daily-snapshots" + region = var.region + + snapshot_schedule_policy { + schedule { + daily_schedule { + days_in_cycle = 1 + # UTC. Daily schedules must start on a multiple of four hours (00, 04, + # 08, ...); the provider only checks HH:00, so a bad value fails at apply. + start_time = "04:00" + } + } + + retention_policy { + max_retention_days = 14 + on_source_disk_delete = "KEEP_AUTO_SNAPSHOTS" + } + } +} + +resource "google_compute_disk_resource_policy_attachment" "pds_snapshots" { + name = google_compute_resource_policy.pds_snapshots.name + disk = google_compute_disk.pds_data.name + zone = var.zone +} + +resource "google_compute_instance" "pds" { + name = "atproto-pds" + machine_type = var.machine_type + zone = var.zone + tags = ["atproto-pds"] + labels = local.labels + + deletion_protection = true + allow_stopping_for_update = true + + boot_disk { + initialize_params { + image = "debian-cloud/debian-12" + type = "pd-balanced" + size = 10 + } + } + + attached_disk { + source = google_compute_disk.pds_data.id + # Surfaces in the guest as /dev/disk/by-id/google-pds-data (the startup + # script mounts it by that path). + device_name = "pds-data" + } + + network_interface { + network = var.network + + access_config { + nat_ip = google_compute_address.pds.address + } + } + + service_account { + email = google_service_account.pds.email + # Broad scope on purpose: real authorization is the IAM granted in + # iam.tf, not OAuth scopes. + scopes = ["cloud-platform"] + } + + # metadata (not metadata_startup_script): the latter forces VM replacement on + # any script change, which deletion_protection would turn into a failed + # apply. As metadata, a script change is an in-place update that takes + # effect on the next boot. + metadata = { + startup-script = templatefile("${path.module}/templates/pds-startup.sh.tftpl", { + project_id = var.project_id + pds_hostname = var.pds_hostname + pds_image = local.pds_image + caddy_image = local.caddy_image + pds_secrets = local.pds_secrets + pds_crawlers = var.pds_crawlers + blob_bucket = local.blob_bucket + }) + } +} diff --git a/infra/gcp/providers.tf b/infra/gcp/providers.tf new file mode 100644 index 000000000..9ff0ef101 --- /dev/null +++ b/infra/gcp/providers.tf @@ -0,0 +1,5 @@ +provider "google" { + project = var.project_id + region = var.region + zone = var.zone +} diff --git a/infra/gcp/scripts/bootstrap.sh b/infra/gcp/scripts/bootstrap.sh new file mode 100755 index 000000000..1ac2770f3 --- /dev/null +++ b/infra/gcp/scripts/bootstrap.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# One-time, per environment, before the first terraform apply. Idempotent. +# Editor is enough. Usage: infra/gcp/scripts/bootstrap.sh +set -euo pipefail +cd "$(dirname "$0")/.." +. scripts/tfvar.sh +env=${1:?usage: bootstrap.sh } +project=$(tfvar "$env" project_id) +region=$(tfvar "$env" region) +bucket=$(sed -n 's/^bucket *= *"\(.*\)".*/\1/p' "envs/$env.gcs.tfbackend") + +echo "== APIs (Compute, Run, Artifact Registry, Secret Manager are on already)" +gcloud services enable dns.googleapis.com monitoring.googleapis.com --project="$project" + +echo "== state bucket gs://$bucket" +if gcloud storage buckets describe "gs://$bucket" --project="$project" >/dev/null 2>&1; then + echo "exists" +else + gcloud storage buckets create "gs://$bucket" --project="$project" --location="$region" \ + --uniform-bucket-level-access --public-access-prevention +fi +# Outside the create branch on purpose: a run that died between create and +# this line would otherwise leave the bucket unversioned for good. +gcloud storage buckets update "gs://$bucket" --versioning >/dev/null +echo "versioning on" + +echo "== Firestore location (the region above should sit inside it)" +# Informational: a project with no Firestore database yet must not fail a +# bootstrap that has otherwise finished. +gcloud firestore databases describe --project="$project" --format='value(locationId)' || true +echo "bootstrapped $env. Next: terraform -chdir=infra/gcp init -backend-config=envs/$env.gcs.tfbackend" diff --git a/infra/gcp/scripts/secrets.sh b/infra/gcp/scripts/secrets.sh new file mode 100755 index 000000000..e6e5c0800 --- /dev/null +++ b/infra/gcp/scripts/secrets.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# After terraform apply: mint the PDS's five secret versions. The VM's startup +# script retries every 3 minutes until they exist (pds-startup-retry.timer), +# so nothing here reboots or resets anything. Values go straight into Secret +# Manager; nothing is printed, written to disk here, or kept in shell history. +# Idempotent: a secret that already has a version is left alone, so re-running +# never rotates anything by accident. Editor plus storage.hmacKeys.create. +# Needs the root initialised (terraform init) — the ids and the service +# account come from its outputs. Usage: infra/gcp/scripts/secrets.sh +set -euo pipefail +cd "$(dirname "$0")/.." +. scripts/tfvar.sh +env=${1:?usage: secrets.sh } +project=$(tfvar "$env" project_id) +sa=$(terraform output -raw pds_service_account) +instance=$(terraform output -json pds_instance) +secrets=$(terraform output -json pds_secrets) +# The outputs come from whichever backend was last `terraform init`ed, the +# project from the argument. Mixed up, this mints one environment's secrets +# against the other environment's service account. Refuse rather than half-run. +case "$sa" in + *"@$project.iam.gserviceaccount.com") ;; + *) + echo "secrets.sh: the initialised root's service account is $sa, but env $env is $project." >&2 + echo " terraform -chdir=infra/gcp init -reconfigure -backend-config=envs/$env.gcs.tfbackend" >&2 + exit 1 + ;; +esac +# id : its Secret Manager id, from locals.pds_secrets. +id() { printf %s "$secrets" | python3 -c 'import json,sys; print(json.load(sys.stdin)[sys.argv[1]], end="")' "$1"; } + +# "no enabled version" and "could not ask" are NOT the same answer: swallowing +# the second one turns an expired login or a missing secretmanager.versions.list +# into a silent re-mint of every secret below, and re-minting the PLC rotation +# key strands the DID document's rotationKeys. Every caller runs in the main +# shell, so exiting here stops the run. +has_version() { + local out + if ! out=$(gcloud secrets versions list "$1" --project="$project" --filter='state=enabled' --format='value(name)' --limit=1); then + echo "secrets.sh: cannot list versions of $1; refusing to mint one blindly" >&2 + exit 1 + fi + [ -n "$out" ] +} +add() { tr -d '\n' | gcloud secrets versions add "$1" --project="$project" --data-file=- >/dev/null; echo "added $1"; } +# ensure : a version from the command's stdout, unless +# one exists already. The generator runs to completion BEFORE add: piped into +# it, a generator that died would still hand add its (empty) stdout, and the +# empty version it wrote would then read as "has a version" for good. +ensure() { + if has_version "$1"; then echo "kept $1 (has a version)"; return; fi + local value + value=$("${@:2}") + [ -n "$value" ] || { echo "secrets.sh: $2 produced nothing for $1" >&2; exit 1; } + printf %s "$value" | add "$1" +} +# Same shapes as the upstream installer.sh. The rotation key is a raw +# secp256k1 scalar as 64 hex chars; the PDS refuses to boot otherwise. +rotation_key() { openssl ecparam -name secp256k1 -genkey -noout -outform DER | tail -c +8 | head -c 32 | xxd -p -c 32; } + +ensure "$(id PDS_ADMIN_PASSWORD)" openssl rand -hex 16 +ensure "$(id PDS_JWT_SECRET)" openssl rand -hex 16 +ensure "$(id PDS_PLC_ROTATION_KEY_K256_PRIVATE_KEY_HEX)" rotation_key + +# Blob bucket HMAC key for the PDS service account; the secret is shown once, +# so both halves go from the JSON straight into Secret Manager, id first. A +# run that died between the two adds left a key whose secret half nobody +# holds: retire it by the id we did store, then mint a fresh pair, so the +# account never accumulates orphaned keys. +key_id=$(id PDS_BLOBSTORE_S3_ACCESS_KEY_ID) +key_secret=$(id PDS_BLOBSTORE_S3_SECRET_ACCESS_KEY) +if has_version "$key_id" && has_version "$key_secret"; then + echo "kept $key_id (has a version)"; echo "kept $key_secret (has a version)" +else + if has_version "$key_id"; then + orphan=$(gcloud secrets versions access latest --secret="$key_id" --project="$project") + echo "retiring HMAC key $orphan (stored id without its secret)" + gcloud storage hmac update "$orphan" --project="$project" --deactivate >/dev/null + gcloud storage hmac delete "$orphan" --project="$project" >/dev/null + fi + hmac=$(gcloud storage hmac create "$sa" --project="$project" --format=json) + printf %s "$hmac" | python3 -c 'import json,sys; print(json.load(sys.stdin)["metadata"]["accessId"], end="")' | add "$key_id" + printf %s "$hmac" | python3 -c 'import json,sys; print(json.load(sys.stdin)["secret"], end="")' | add "$key_secret" + unset hmac +fi + +name=$(printf %s "$instance" | python3 -c 'import json,sys; print(json.load(sys.stdin)["name"], end="")') +zone=$(printf %s "$instance" | python3 -c 'import json,sys; print(json.load(sys.stdin)["zone"], end="")') +echo "== $name picks the secrets up within 3 minutes (pds-startup-retry.timer)" +echo "watch: gcloud compute instances get-serial-port-output $name --project=$project --zone=$zone | grep pds-startup" diff --git a/infra/gcp/scripts/tfvar.sh b/infra/gcp/scripts/tfvar.sh new file mode 100755 index 000000000..c404ca28a --- /dev/null +++ b/infra/gcp/scripts/tfvar.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# tfvar : the value of a string variable as Terraform will see it — +# envs/.tfvars first, else the variable's default in variables.tf. Sourced +# by bootstrap.sh and secrets.sh; run from infra/gcp. +tfvar() { + local v + v=$(sed -n "s/^$2 *= *\"\\(.*\\)\".*/\\1/p" "envs/$1.tfvars") + [ -n "$v" ] || v=$(sed -n "/^variable \"$2\" {/,/^}/s/^ *default *= *\"\\(.*\\)\".*/\\1/p" variables.tf) + [ -n "$v" ] || { echo "tfvar: no value for $2 in envs/$1.tfvars or variables.tf" >&2; return 1; } + printf %s "$v" +} diff --git a/infra/gcp/secrets.tf b/infra/gcp/secrets.tf new file mode 100644 index 000000000..381106572 --- /dev/null +++ b/infra/gcp/secrets.tf @@ -0,0 +1,13 @@ +# Secret RESOURCES only — never versions. Terraform state stores values in +# plaintext, so secret material is added out of band (ADR 0001): +# gcloud secrets versions add --data-file=- +resource "google_secret_manager_secret" "pds" { + for_each = toset(values(local.pds_secrets)) + + secret_id = each.value + labels = local.labels + + replication { + auto {} + } +} diff --git a/infra/gcp/templates/pds-startup.sh.tftpl b/infra/gcp/templates/pds-startup.sh.tftpl new file mode 100644 index 000000000..5f20cdff4 --- /dev/null +++ b/infra/gcp/templates/pds-startup.sh.tftpl @@ -0,0 +1,265 @@ +#!/usr/bin/env bash +# Rendered by Terraform (pds.tf) into the VM's startup-script metadata. Runs on +# every boot, and again every few minutes (pds-startup-retry.timer, installed +# below) until /run/pds/pds.env exists; every step is idempotent. Output lands +# in the google-startup-scripts.service journal and the serial console. +# +# Template variables (project_id, pds_hostname, pds_image, caddy_image, +# pds_crawlers, blob_bucket, pds_secrets) are interpolated by Terraform; bare +# $VARS are shell. +set -euo pipefail + +# One run at a time: the retry timer must not race a slow first boot (apt, +# mkfs). A run that finds the lock held has nothing to do — the holder fetches +# the secrets itself. +exec 9>/run/lock/pds-startup +if ! flock -n 9; then + echo "pds-startup: another run holds the lock; skipping" + exit 0 +fi + +# --- Docker ------------------------------------------------------------------ +# DPkg::Lock::Timeout: the image's apt-daily / unattended-upgrades timers race +# a first boot for the dpkg lock; without it, `set -e` ends the run before the +# data disk is mounted or the unit exists. +APT="apt-get -o DPkg::Lock::Timeout=600" +if ! command -v docker >/dev/null 2>&1; then + $APT update + $APT install -y ca-certificates curl + install -m 0755 -d /etc/apt/keyrings + curl -fsSL --retry 5 https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc + echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian $(. /etc/os-release && echo "$VERSION_CODENAME") stable" \ + > /etc/apt/sources.list.d/docker.list + $APT update + $APT install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin +fi + +# --- Data disk --------------------------------------------------------------- +DISK=/dev/disk/by-id/google-pds-data +if [ ! -b "$DISK" ]; then + echo "pds-startup: data disk $DISK is not attached" + exit 1 +fi +# Format ONLY when blkid positively reports "no filesystem" (exit 2). Exit 0 +# with a type means data; anything else is a failed probe, and formatting on +# a failed probe is how a disk full of data gets wiped. This guard is what +# makes reboots and re-runs safe for the PDS's data. +if fstype=$(blkid -o value -s TYPE "$DISK"); then + if [ -z "$fstype" ]; then + echo "pds-startup: $DISK has signatures but no filesystem type; refusing to format" + exit 1 + fi +elif [ $? -eq 2 ]; then + mkfs.ext4 -m 0 -E lazy_itable_init=0,lazy_journal_init=0,discard "$DISK" +else + echo "pds-startup: blkid could not probe $DISK; refusing to format" + exit 1 +fi +mkdir -p /pds +if ! grep -q "^$DISK " /etc/fstab; then + echo "$DISK /pds ext4 discard,defaults,nofail 0 2" >> /etc/fstab +fi +mountpoint -q /pds || mount /pds +mkdir -p /pds/data /pds/caddy +# Secrets are rendered into /run (tmpfs): regenerated from Secret Manager on +# every boot, never on the data disk, never in its snapshots. (dockerd keeps a +# copy of a container's env in /var/lib/docker on the boot disk, which is not +# snapshotted — see the README's rule.) +install -d -m 0700 /run/pds + +# --- Compose stack definition ------------------------------------------------ +# Single-host certificate via HTTP-01. If user handles under this hostname are +# ever wanted, that means a wildcard cert via DNS-01 and a caddy dns plugin — +# an identity decision, not a startup-script edit. +cat > /pds/Caddyfile < /pds/compose.yaml < /etc/systemd/system/pds.service <<'EOF' +[Unit] +Description=MAPLE atproto PDS (caddy + pds via docker compose) +Requires=docker.service +After=docker.service +# /run is empty at boot; this script writes the env file after fetching the +# secrets and then starts the unit. Without the condition the unit would race +# the script at boot and fail on the missing env_file. +ConditionPathExists=/run/pds/pds.env + +[Service] +Type=oneshot +RemainAfterExit=yes +WorkingDirectory=/pds +ExecStart=/usr/bin/docker compose up -d --remove-orphans +ExecStop=/usr/bin/docker compose down + +[Install] +WantedBy=multi-user.target +EOF + +# Re-run this script until the env file exists: covers secrets minted after +# the VM came up (secrets.sh) and a Secret Manager blip at boot, with no +# reboot and no human in the loop. The condition switches it off afterwards. +cat > /etc/systemd/system/pds-startup-retry.service <<'EOF' +[Unit] +Description=Re-run the PDS startup script until its secrets are available +ConditionPathExists=!/run/pds/pds.env + +[Service] +Type=oneshot +# No timeout: a retry that lands on a boot where the docker install never +# finished has to redo it, and systemd's 90s default would SIGTERM apt +# mid-transaction, every 3 minutes. The flock at the top is what bounds +# concurrency here, not a timeout. +TimeoutStartSec=0 +ExecStart=/usr/bin/google_metadata_script_runner startup +EOF +cat > /etc/systemd/system/pds-startup-retry.timer <<'EOF' +[Unit] +Description=Retry the PDS startup script every 3 minutes until it has its secrets + +[Timer] +OnBootSec=3min +OnUnitActiveSec=3min + +[Install] +WantedBy=timers.target +EOF +systemctl daemon-reload +systemctl enable pds.service +systemctl enable --now pds-startup-retry.timer + +# --- Secrets, then start ----------------------------------------------------- +TOKEN=$(curl -sSf --retry 5 --retry-all-errors --retry-delay 2 -H "Metadata-Flavor: Google" \ + "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token" \ + | python3 -c 'import json,sys; print(json.load(sys.stdin)["access_token"])') + +# Prints the secret on stdout. On failure, logs the HTTP status so a 404 (no +# version yet: the designed out-of-band state), a 403 (iam.tf's accessor +# grants not applied yet) and a network failure (000) read differently. +fetch_secret() { + local body code + body=$(mktemp) + code=$(curl -sS --retry 5 --retry-all-errors --retry-delay 2 -o "$body" -w '%%{http_code}' \ + -H "Authorization: Bearer $TOKEN" \ + "https://secretmanager.googleapis.com/v1/projects/${project_id}/secrets/$1/versions/latest:access") || code=000 + if [ "$code" = 200 ]; then + python3 -c 'import base64,json,sys; print(base64.b64decode(json.load(sys.stdin)["payload"]["data"]).decode(), end="")' < "$body" + rm -f "$body" + return 0 + fi + echo "pds-startup: secret $1: HTTP $code $(tr -d '\n' < "$body" | head -c 200)" >&2 + rm -f "$body" + return "$([ "$code" = 404 ] && echo 44 || echo 1)" +} + +# Fetch each secret exactly once; the missing-check guards the same bytes that +# get written. (An assignment-with-substitution keeps the fetch's exit status +# visible, unlike a substitution inside echo. `|| rc=$?` and not an if/fi: the +# exit status of an `if` whose condition was false is 0, not the condition's, +# so reading $? after the fi would score every gap as a hard failure.) +declare -A secret +missing=0 +failed=0 +for s in ${join(" ", values(pds_secrets))}; do + rc=0 + secret[$s]=$(fetch_secret "$s") || rc=$? + if [ "$rc" -ne 0 ]; then + missing=1 + [ "$rc" -eq 44 ] || failed=1 + fi +done +if [ "$missing" -ne 0 ]; then + # Not starting the PDS. pds-startup-retry.timer runs this script again in 3 + # minutes; no reboot needed. Exit status: 0 when every gap is "no version + # yet" (secrets.sh not run), non-zero when something is actually broken. + echo "pds-startup: not starting pds; retrying in 3 minutes" + exit "$failed" +fi + +umask 077 +{ + echo "PDS_HOSTNAME=${pds_hostname}" + echo "PDS_PORT=3000" + echo "PDS_DID_PLC_URL=https://plc.directory" + echo "PDS_DATA_DIRECTORY=/pds/data" + echo "LOG_ENABLED=true" + # Blobs go to GCS through its S3-compatible XML API. GCS accepts AWS-style + # SigV4 with any region string; "auto" is what Google's own S3-SDK samples + # pass. Path-style keeps the bucket out of the TLS hostname. + echo "PDS_BLOBSTORE_S3_BUCKET=${blob_bucket}" + echo "PDS_BLOBSTORE_S3_ENDPOINT=https://storage.googleapis.com" + echo "PDS_BLOBSTORE_S3_REGION=auto" + echo "PDS_BLOBSTORE_S3_FORCE_PATH_STYLE=true" + # AWS SDK v3 >= 3.729 (the PDS bundles a newer one) sends CRC32 "flexible + # checksums" on every PUT by default. GCS's XML API does not implement them + # and rejects the signature, so uploads 4xx while /xrpc/_health stays green. + # These are the SDK's documented env overrides; CRC32 only when the request + # type requires it, which no S3 blobstore call does. + echo "AWS_REQUEST_CHECKSUM_CALCULATION=when_required" + echo "AWS_RESPONSE_CHECKSUM_VALIDATION=when_required" +%{ if pds_crawlers != "" ~} + echo "PDS_CRAWLERS=${pds_crawlers}" +%{ endif ~} + echo "PDS_INVITE_REQUIRED=true" +%{ for env_name, secret_id in pds_secrets ~} + echo "${env_name}=$${secret[${secret_id}]}" +%{ endfor ~} +} > /run/pds/pds.env + +systemctl restart pds.service + +# --- Ops Agent --------------------------------------------------------------- +# Host metrics the default GCE set lacks (memory, filesystem usage), for the +# disk and memory alerts in monitoring-pds.tf; its two roles are already +# granted in iam.tf. After the PDS start on purpose: nothing above needs it, +# and a monitoring nicety must never keep the PDS down, so a failure here is +# logged and left for the next boot rather than ending the run. Google's +# script adds the apt repo and refreshes it (waiting for the dpkg lock +# itself); the install goes through $APT like docker's so a wedged lock fails +# after 600s instead of hanging. Default agent config: host metrics and +# syslog; container logs stay on the box (compose caps them above). +if ! dpkg -s google-cloud-ops-agent >/dev/null 2>&1; then + if ! { curl -fsSL --retry 5 https://dl.google.com/cloudagents/add-google-cloud-ops-agent-repo.sh \ + -o /run/add-ops-agent-repo.sh \ + && bash /run/add-ops-agent-repo.sh \ + && $APT install -y google-cloud-ops-agent; }; then + echo "pds-startup: ops agent install failed; no disk/memory metrics until the next boot" + fi +fi diff --git a/infra/gcp/variables.tf b/infra/gcp/variables.tf new file mode 100644 index 000000000..7b0284e0e --- /dev/null +++ b/infra/gcp/variables.tf @@ -0,0 +1,90 @@ +variable "env" { + description = "Environment name (dev or prod). Used for labels only — isolation comes from the separate GCP projects." + type = string +} + +variable "project_id" { + description = "GCP project id (digital-testimony-dev / digital-testimony-prod)." + type = string +} + +variable "region" { + description = "Region for regional resources. Confirm it matches the project's Firestore location at bootstrap (see README)." + type = string + default = "us-central1" +} + +variable "zone" { + description = "Zone for the PDS VM and its data disk. Must sit inside region: the snapshot policy is regional and attaches to the zonal disk." + type = string + default = "us-central1-a" + + validation { + condition = startswith(var.zone, "${var.region}-") + error_message = "zone must be a zone of region (e.g. region us-central1, zone us-central1-a)." + } +} + +variable "network" { + description = "VPC network for the PDS VM and firewall rules." + type = string + default = "default" +} + +variable "pds_hostname" { + description = "Public hostname of the PDS, and the dns_name of the delegated Cloud DNS zone. APPLY-ONCE: it gets baked into the DID document at account creation and is effectively immutable afterwards. Enforced by prevent_destroy on the zone (dns.tf): a change plans as a replacement and the apply refuses it." + type = string +} + +variable "machine_type" { + description = "PDS VM machine type (ADR 0001)." + type = string + default = "e2-small" +} + +variable "data_disk_size_gb" { + description = "Size of the PDS data disk (SQLite databases + actor store; blobs live in GCS, see blobs.tf)." + type = number + default = 20 +} + +variable "pds_crawlers" { + description = "Comma-separated relay URLs the PDS asks to crawl it on boot (PDS_CRAWLERS). Empty = the PDS is not announced to any relay: nothing downstream of a relay ever sees its repos. Set per environment on purpose." + type = string + default = "https://bsky.network" +} + +variable "ci_planner" { + description = "IAM member (serviceAccount:…) whose key the terraform-plan GitHub environment holds. Gets read access to this environment's state bucket and to the parent DNS zone (dns.tf), nothing else: enough for `plan -refresh=false` on PRs, not enough to apply. null = no CI plan for this environment." + type = string + default = null + + validation { + condition = var.ci_planner == null || can(regex("^serviceAccount:[^:]+$", var.ci_planner)) + error_message = "ci_planner must be a serviceAccount: member." + } +} + +# Only channel types whose labels hold no secret are built here: a PagerDuty +# service key or Slack token would land in state, which ADR 0001 forbids. +# When prod picks such a pager, extend this with an optional token secret per +# channel fed through the provider's write-only service_key_wo / auth_token_wo +# from an ephemeral Secret Manager read: the same out-of-band pattern as +# secrets.sh, and the token never enters state. +variable "alert_channels" { + description = "Where every alert policy in this root notifies, keyed by a short name: the Cloud Monitoring channel type and its labels (email: email_address; pubsub: topic; sms: number). Per environment on purpose: dev must never page whoever is on call for prod. A pager goes in as its email-integration address; a channel-type swap here changes no policy." + type = map(object({ + type = string + labels = map(string) + })) + + validation { + condition = length(var.alert_channels) > 0 + error_message = "at least one alert channel: a policy that notifies nobody looks monitored and is not." + } + + validation { + condition = alltrue([for c in values(var.alert_channels) : contains(["email", "pubsub", "sms"], c.type)]) + error_message = "alert_channels types are email, pubsub or sms: the ones whose labels hold no secret (ADR 0001). A token-bearing type needs the write-only path described above the variable." + } +} diff --git a/infra/gcp/versions.tf b/infra/gcp/versions.tf new file mode 100644 index 000000000..b07c611af --- /dev/null +++ b/infra/gcp/versions.tf @@ -0,0 +1,14 @@ +# Keep terraform_version in .github/workflows/terraform-checks.yml in lockstep +# with required_version here. +terraform { + required_version = "~> 1.16.0" + + required_providers { + google = { + source = "hashicorp/google" + # v8.0.0 was released 2026-08-26; staying on the mature 7.x line until + # v8 has settled. Bump deliberately. + version = "~> 7.46" + } + } +}