From 73f50b3a68ec61abec844593b36b85f2618f317f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Fri, 10 Jul 2026 16:38:53 +0200 Subject: [PATCH 01/24] PoC of blue-green deployer to dedicated host --- tf/modules/ooniapi_service_deployer/main.tf | 258 +++++++++++++++++- .../templates/buildspec_deploy.yml | 92 +++++++ .../templates/nginx_upstream.conf.tftpl | 4 + .../templates/quadlet.container.tftpl | 23 ++ .../ooniapi_service_deployer/variables.tf | 105 ++++++- 5 files changed, 463 insertions(+), 19 deletions(-) create mode 100644 tf/modules/ooniapi_service_deployer/templates/buildspec_deploy.yml create mode 100644 tf/modules/ooniapi_service_deployer/templates/nginx_upstream.conf.tftpl create mode 100644 tf/modules/ooniapi_service_deployer/templates/quadlet.container.tftpl diff --git a/tf/modules/ooniapi_service_deployer/main.tf b/tf/modules/ooniapi_service_deployer/main.tf index fb32fefd..b95722b7 100755 --- a/tf/modules/ooniapi_service_deployer/main.tf +++ b/tf/modules/ooniapi_service_deployer/main.tf @@ -161,6 +161,209 @@ resource "aws_codebuild_project" "ooniapi" { } } +## Podman Quadlet blue/green deploy (deploy_mode = "blue_green") + +resource "aws_s3_object" "quadlet_unit" { + for_each = var.deploy_mode == "blue_green" ? { a = var.host_port_a, b = var.host_port_b } : {} + + bucket = var.quadlet_units_bucket + key = "${var.service_name}/${var.service_name}-${each.key}.container" + content_type = "text/plain" + + content = templatefile("${path.module}/templates/quadlet.container.tftpl", { + service_name = var.service_name + slot = each.key + host_port = each.value + container_port = var.container_port + network_name = var.network_name + env_vars = var.env_vars + secrets = var.secrets + }) +} + +resource "aws_s3_object" "nginx_upstream" { + count = var.deploy_mode == "blue_green" ? 1 : 0 + + bucket = var.quadlet_units_bucket + key = "${var.service_name}/${var.service_name}-upstream.conf" + content_type = "text/plain" + + content = templatefile("${path.module}/templates/nginx_upstream.conf.tftpl", { + service_name = var.service_name + host_port_a = var.host_port_a + host_port_b = var.host_port_b + }) +} + +resource "aws_iam_policy" "deploy" { + count = var.deploy_mode == "blue_green" ? 1 : 0 + + description = "Policy used in trust relationship with the blue/green deploy CodeBuild project" + name = "codebuild-deploy-${var.service_name}-${var.aws_region}" + path = "/service-role/" + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents" + ] + Resource = [ + "arn:aws:logs:${var.aws_region}:${local.account_id}:log-group:/aws/codebuild/ooniapi-${var.service_name}-deploy", + "arn:aws:logs:${var.aws_region}:${local.account_id}:log-group:/aws/codebuild/ooniapi-${var.service_name}-deploy:*" + ] + }, + { + # required for CodeBuild to read the CodePipeline BuildArtifact + # (imagedefinitions.json), mirrors the grant on the build role + Effect = "Allow" + Action = [ + "s3:GetObject", + "s3:GetObjectVersion", + "s3:GetBucketAcl", + "s3:GetBucketLocation" + ] + Resource = [ + "arn:aws:s3:::${var.codepipeline_bucket}", + "arn:aws:s3:::${var.codepipeline_bucket}/*" + ] + }, + { + Effect = "Allow" + Action = ["s3:GetObject"] + Resource = ["arn:aws:s3:::${var.quadlet_units_bucket}/${var.service_name}/*"] + }, + { + Effect = "Allow" + Action = ["secretsmanager:GetSecretValue"] + Resource = [ + var.deploy_ssh_key_secret_arn, + var.service_secrets_arn + ] + } + ] + }) +} + +resource "aws_iam_role" "deploy" { + count = var.deploy_mode == "blue_green" ? 1 : 0 + + assume_role_policy = </dev/null || (apt-get update -y && apt-get install -y jq) + + pre_build: + commands: + - aws secretsmanager get-secret-value --secret-id "$DEPLOY_SSH_KEY_SECRET_ARN" --query SecretString --output text > /tmp/deploy_key + - chmod 600 /tmp/deploy_key + - aws secretsmanager get-secret-value --secret-id "$SERVICE_SECRETS_ARN" --query SecretString --output text > /tmp/service_secrets.json + - IMAGE_URI=$(jq -r '.[0].imageUri' imagedefinitions.json) + - export IMAGE_TAG="${IMAGE_URI##*:}" + - echo "Deploying $SERVICE_NAME image tag $IMAGE_TAG to $DEPLOY_HOST_PRIMARY then $DEPLOY_HOST_SECONDARY" + + build: + commands: + - | + set -euo pipefail + + deploy_host() { + HOST="$1" + echo "=== $SERVICE_NAME: deploying to $HOST ===" + + ACTIVE_SLOT=$(ssh $SSH_OPTS "${DEPLOY_SSH_USER}@${HOST}" "cat /etc/ooniapi/${SERVICE_NAME}/active_slot 2>/dev/null || echo a") + if [ "$ACTIVE_SLOT" = "a" ]; then + TARGET_SLOT=b + TARGET_PORT="$HOST_PORT_B" + else + TARGET_SLOT=a + TARGET_PORT="$HOST_PORT_A" + fi + echo "$SERVICE_NAME on $HOST: active slot is $ACTIVE_SLOT, deploying to slot $TARGET_SLOT (port $TARGET_PORT)" + + # a. + b. render/fetch the target slot's quadlet unit and install it + UNIT_FILE="${SERVICE_NAME}-${TARGET_SLOT}.container" + aws s3 cp "s3://${QUADLET_BUCKET}/${SERVICE_NAME}/${UNIT_FILE}" "/tmp/${UNIT_FILE}" + sed -i "s|__IMAGE_TAG__|${IMAGE_TAG}|g" "/tmp/${UNIT_FILE}" + + scp $SSH_OPTS "/tmp/${UNIT_FILE}" "${DEPLOY_SSH_USER}@${HOST}:/tmp/${UNIT_FILE}" + ssh $SSH_OPTS "${DEPLOY_SSH_USER}@${HOST}" "sudo mv /tmp/${UNIT_FILE} /etc/containers/systemd/${UNIT_FILE}" + + # c. push service secrets as podman secrets (values travel over + # stdin, never interpolated into the remote command string) + for KEY in $(jq -r 'keys[]' /tmp/service_secrets.json); do + VALUE=$(jq -r --arg k "$KEY" '.[$k]' /tmp/service_secrets.json) + printf '%s' "$VALUE" | ssh $SSH_OPTS "${DEPLOY_SSH_USER}@${HOST}" "podman secret create --replace ${KEY} -" + done + + # d. reload systemd and (re)start only the target slot + ssh $SSH_OPTS "${DEPLOY_SSH_USER}@${HOST}" "sudo systemctl daemon-reload && sudo systemctl restart ${SERVICE_NAME}-${TARGET_SLOT}.service" + + # e. health-check the target slot directly, bypassing nginx + HEALTHY="" + for i in $(seq 1 10); do + if ssh $SSH_OPTS "${DEPLOY_SSH_USER}@${HOST}" "curl -sf -o /dev/null http://127.0.0.1:${TARGET_PORT}/health"; then + HEALTHY=1 + break + fi + sleep 2 + done + if [ -z "$HEALTHY" ]; then + echo "$SERVICE_NAME on $HOST: slot $TARGET_SLOT failed health check, aborting deploy" + exit 1 + fi + + # f. flip nginx to the target slot and record the new active slot + if [ "$TARGET_SLOT" = "a" ]; then + STATE_A="" + STATE_B="down" + else + STATE_A="down" + STATE_B="" + fi + UPSTREAM_FILE="${SERVICE_NAME}-upstream.conf" + aws s3 cp "s3://${QUADLET_BUCKET}/${SERVICE_NAME}/${UPSTREAM_FILE}" "/tmp/${UPSTREAM_FILE}" + sed -i "s|__STATE_A__|${STATE_A}|g; s|__STATE_B__|${STATE_B}|g" "/tmp/${UPSTREAM_FILE}" + + scp $SSH_OPTS "/tmp/${UPSTREAM_FILE}" "${DEPLOY_SSH_USER}@${HOST}:/tmp/${UPSTREAM_FILE}" + ssh $SSH_OPTS "${DEPLOY_SSH_USER}@${HOST}" "sudo mv /tmp/${UPSTREAM_FILE} /etc/nginx/conf.d/${UPSTREAM_FILE} && sudo nginx -t && sudo systemctl reload nginx" + ssh $SSH_OPTS "${DEPLOY_SSH_USER}@${HOST}" "echo ${TARGET_SLOT} | sudo tee /etc/ooniapi/${SERVICE_NAME}/active_slot > /dev/null" + + echo "=== $SERVICE_NAME on $HOST: now serving from slot $TARGET_SLOT ===" + } + + deploy_host "$DEPLOY_HOST_PRIMARY" + deploy_host "$DEPLOY_HOST_SECONDARY" diff --git a/tf/modules/ooniapi_service_deployer/templates/nginx_upstream.conf.tftpl b/tf/modules/ooniapi_service_deployer/templates/nginx_upstream.conf.tftpl new file mode 100644 index 00000000..d239d393 --- /dev/null +++ b/tf/modules/ooniapi_service_deployer/templates/nginx_upstream.conf.tftpl @@ -0,0 +1,4 @@ +upstream ${service_name} { + server 127.0.0.1:${host_port_a} __STATE_A__; + server 127.0.0.1:${host_port_b} __STATE_B__; +} diff --git a/tf/modules/ooniapi_service_deployer/templates/quadlet.container.tftpl b/tf/modules/ooniapi_service_deployer/templates/quadlet.container.tftpl new file mode 100644 index 00000000..f309d8c4 --- /dev/null +++ b/tf/modules/ooniapi_service_deployer/templates/quadlet.container.tftpl @@ -0,0 +1,23 @@ +[Unit] +Description=${service_name} (slot ${slot}) +After=network-online.target +Wants=network-online.target + +[Container] +ContainerName=${service_name}-${slot} +Image=docker.io/ooni/api-${service_name}:__IMAGE_TAG__ +Network=${network_name} +PublishPort=127.0.0.1:${host_port}:${container_port} +%{ for key, value in env_vars ~} +Environment="${key}=${value}" +%{ endfor ~} +%{ for secret_name in secrets ~} +Secret=${secret_name} +%{ endfor ~} + +[Service] +Restart=always +TimeoutStartSec=60 + +[Install] +WantedBy=multi-user.target diff --git a/tf/modules/ooniapi_service_deployer/variables.tf b/tf/modules/ooniapi_service_deployer/variables.tf index 73e0dc7e..67be2b53 100644 --- a/tf/modules/ooniapi_service_deployer/variables.tf +++ b/tf/modules/ooniapi_service_deployer/variables.tf @@ -30,15 +30,112 @@ variable "trigger_path" { description = "path filter for push changes which trigger the codepipeline eg. ooniapi/services/oonirun/**" } +variable "environment" { + description = "Deployment environment (e.g., prod, dev)" + type = string +} + +variable "deploy_mode" { + description = <<-EOF + Which Deploy stage implementation the pipeline uses: + - "ecs" (default) the existing ECS rolling-deploy stage. + - "blue_green" Podman Quadlet blue/green deploy to dedicated Hetzner + hosts, driven by a CodeBuild "Deploy" action over SSH. + This is opt-in per service so unmigrated services keep working unchanged. + EOF + type = string + default = "ecs" + + validation { + condition = contains(["ecs", "blue_green"], var.deploy_mode) + error_message = "deploy_mode must be either \"ecs\" or \"blue_green\"." + } +} + +# --- deploy_mode = "ecs" ----------------------------------------------- + variable "ecs_cluster_name" { - description = "id of the cluster to deploy into" + description = "id of the cluster to deploy into. Required when deploy_mode = \"ecs\"." + type = string + default = null } variable "ecs_service_name" { - description = "id of the service in the cluster to deploy" + description = "id of the service in the cluster to deploy. Required when deploy_mode = \"ecs\"." + type = string + default = null } -variable "environment" { - description = "Deployment environment (e.g., prod, dev)" +# --- deploy_mode = "blue_green" ----------------------------------------- + +variable "quadlet_units_bucket" { + description = "S3 bucket that rendered Quadlet unit files and the nginx upstream conf snippet are uploaded to. Required when deploy_mode = \"blue_green\"." + type = string + default = null +} + +variable "host_port_a" { + description = "Host port bound to the \"a\" deploy slot. Required when deploy_mode = \"blue_green\"." + type = number + default = null +} + +variable "host_port_b" { + description = "Host port bound to the \"b\" deploy slot. Required when deploy_mode = \"blue_green\"." + type = number + default = null +} + +variable "container_port" { + description = "Port the service listens on inside the container. Required when deploy_mode = \"blue_green\"." + type = number + default = null +} + +variable "network_name" { + description = "Podman network the service's containers attach to. Required when deploy_mode = \"blue_green\"." + type = string + default = null +} + +variable "secrets" { + description = "Names of Podman secrets (created/updated on the host during deploy) to mount into the container. Values live in var.service_secrets_arn." + type = list(string) + default = [] +} + +variable "env_vars" { + description = "Cleartext environment variables for the container. Same shape as ooniapi_service's task_environment (map(string))." + type = map(string) + default = {} +} + +variable "service_secrets_arn" { + description = "ARN of the Secrets Manager secret holding the service's runtime secrets as a flat JSON key/value object. Each key is pushed to the target hosts as a Podman secret during deploy. Required when deploy_mode = \"blue_green\"." + type = string + default = null +} + +variable "deploy_ssh_key_secret_arn" { + description = "ARN of the Secrets Manager secret holding the SSH private key the deploy CodeBuild job uses to reach the target hosts. Required when deploy_mode = \"blue_green\"." + type = string + default = null +} + +variable "deploy_host_primary" { + description = "Hostname/IP of the primary dedicated host to deploy to. Required when deploy_mode = \"blue_green\"." + type = string + default = null +} + +variable "deploy_host_secondary" { + description = "Hostname/IP of the secondary dedicated host to deploy to. Required when deploy_mode = \"blue_green\"." + type = string + default = null +} + +variable "deploy_ssh_user" { + description = "SSH user the deploy job connects as on the target hosts." type = string + default = "deploy" } From d7bcb610b83a93f7b5364507ba502b8148235fe9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Fri, 10 Jul 2026 16:52:02 +0200 Subject: [PATCH 02/24] Add usage of the updated module from dev and prod --- tf/environments/dev/main.tf | 186 +++++++++++++++++++++++++++++++++++ tf/environments/prod/main.tf | 186 +++++++++++++++++++++++++++++++++++ 2 files changed, 372 insertions(+) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 5d1f9e64..943e71a4 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -253,6 +253,96 @@ data "aws_ssm_parameter" "account_id_hashing_key" { name = "/oonidevops/secrets/ooni_services/account_id_hashing_key" } +### Blue/green deploy secrets (Podman Quadlet on dedicated Hetzner hosts) +# +# These mirror the values already passed as `task_secrets` to the ECS task +# definitions above, but consolidated into one JSON blob per service so the +# "blue_green" deploy_mode CodeBuild job can fetch them in a single +# secretsmanager:GetSecretValue call and push each key as a Podman secret. +# Values are pulled from the same underlying data sources used by the ECS +# `task_secrets` maps, so both deploy paths stay in sync automatically. + +data "aws_secretsmanager_secret_version" "ooniapi_user_access_key_id" { + secret_id = module.ooniapi_user.aws_access_key_id_arn +} + +data "aws_secretsmanager_secret_version" "ooniapi_user_secret_access_key" { + secret_id = module.ooniapi_user.aws_secret_access_key_arn +} + +locals { + ooniapi_deploy_service_secrets = { + reverseproxy = { + PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value + } + ooniprobe = { + POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value + JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret_legacy.value + PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_write_url.value + ANONC_SECRET_KEY = data.aws_ssm_parameter.anonc_secret_key.value + } + oonirun = { + POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value + JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value + PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_readonly_url.value + } + oonifindings = { + POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value + JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value + PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_readonly_url.value + } + ooniauth = { + POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value + JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value + PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value + ACCOUNT_ID_HASHING_KEY = data.aws_ssm_parameter.account_id_hashing_key.value + AWS_SECRET_ACCESS_KEY = data.aws_secretsmanager_secret_version.ooniapi_user_secret_access_key.secret_string + AWS_ACCESS_KEY_ID = data.aws_secretsmanager_secret_version.ooniapi_user_access_key_id.secret_string + } + oonimeasurements = { + POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value + JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value + PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_readonly_test_url.value + ACCOUNT_ID_HASHING_KEY = data.aws_ssm_parameter.account_id_hashing_key.value + } + } +} + +resource "aws_secretsmanager_secret" "ooniapi_deploy_service_secrets" { + for_each = local.ooniapi_deploy_service_secrets + + name = "oonidevops/ooniapi/${each.key}/service_secrets" + tags = local.tags +} + +resource "aws_secretsmanager_secret_version" "ooniapi_deploy_service_secrets" { + for_each = local.ooniapi_deploy_service_secrets + + secret_id = aws_secretsmanager_secret.ooniapi_deploy_service_secrets[each.key].id + secret_string = jsonencode(each.value) +} + +# Shared by every service's blue/green deploy job. The private key itself is +# generated and rotated out-of-band (see the deploy README); Terraform only +# owns the secret container, not its value. +resource "aws_secretsmanager_secret" "ooniapi_deploy_ssh_key" { + name = "oonidevops/ooniapi/deploy_ssh_key" + tags = local.tags +} + +resource "aws_secretsmanager_secret_version" "ooniapi_deploy_ssh_key" { + secret_id = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.id + secret_string = "REPLACE_ME: populate out-of-band with the \"deploy\" user's SSH private key" + + lifecycle { + ignore_changes = [secret_string] + } +} + resource "random_id" "artifact_id" { byte_length = 4 } @@ -557,6 +647,10 @@ EOF module "ooniapi_ooniprobe_deployer" { source = "../../modules/ooniapi_service_deployer" + # Flip to "blue_green" to switch this service to the Podman Quadlet + # blue/green deploy on the dedicated Hetzner hosts. + deploy_mode = "ecs" + service_name = "ooniprobe" repo = "ooni/backend" branch_name = "master" @@ -569,6 +663,22 @@ module "ooniapi_ooniprobe_deployer" { ecs_service_name = module.ooniapi_ooniprobe.ecs_service_name ecs_cluster_name = module.ooniapi_cluster.cluster_name + + # Pre-wired for the future flip to deploy_mode = "blue_green" + env_vars = { + FASTPATH_URL = "http://fastpath.${local.environment}.ooni.io:8472" + FASTPATH_URLS = jsonencode([for h in local.fastpath_hosts : "http://${h}:8472"]) + FAILED_REPORTS_BUCKET = aws_s3_bucket.ooniprobe_failed_reports.bucket + COLLECTOR_ID = 3 # use a different one in prod + CONFIG_BUCKET = aws_s3_bucket.ooni_private_config_bucket.bucket + TOR_TARGETS = "tor_targets.json" + PSIPHON_CONFIG = "psiphon_config.json" + ANONC_MANIFEST_BUCKET = aws_s3_bucket.anoncred_manifests.bucket + ANONC_MANIFEST_FILE = "manifest.json" + } + secrets = keys(local.ooniapi_deploy_service_secrets.ooniprobe) + service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["ooniprobe"].arn + deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn } module "ooniapi_ooniprobe" { @@ -634,6 +744,10 @@ module "ooniapi_ooniprobe" { module "ooniapi_reverseproxy_deployer" { source = "../../modules/ooniapi_service_deployer" + # Flip to "blue_green" to switch this service to the Podman Quadlet + # blue/green deploy on the dedicated Hetzner hosts. + deploy_mode = "ecs" + service_name = "reverseproxy" repo = "ooni/backend" branch_name = "master" @@ -646,6 +760,14 @@ module "ooniapi_reverseproxy_deployer" { ecs_service_name = module.ooniapi_reverseproxy.ecs_service_name ecs_cluster_name = module.ooniapi_cluster.cluster_name + + # Pre-wired for the future flip to deploy_mode = "blue_green" + env_vars = { + TARGET_URL = "https://backend-hel.ooni.org/" + } + secrets = keys(local.ooniapi_deploy_service_secrets.reverseproxy) + service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["reverseproxy"].arn + deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn } module "ooniapi_reverseproxy" { @@ -941,6 +1063,10 @@ module "fastpath_builder" { module "ooniapi_oonirun_deployer" { source = "../../modules/ooniapi_service_deployer" + # Flip to "blue_green" to switch this service to the Podman Quadlet + # blue/green deploy on the dedicated Hetzner hosts. + deploy_mode = "ecs" + service_name = "oonirun" repo = "ooni/backend" branch_name = "oonirun-v2-1" @@ -953,6 +1079,12 @@ module "ooniapi_oonirun_deployer" { ecs_service_name = module.ooniapi_oonirun.ecs_service_name ecs_cluster_name = module.ooniapi_cluster.cluster_name + + # Pre-wired for the future flip to deploy_mode = "blue_green" + env_vars = {} + secrets = keys(local.ooniapi_deploy_service_secrets.oonirun) + service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["oonirun"].arn + deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn } module "ooniapi_oonirun" { @@ -992,6 +1124,10 @@ module "ooniapi_oonirun" { module "ooniapi_oonifindings_deployer" { source = "../../modules/ooniapi_service_deployer" + # Flip to "blue_green" to switch this service to the Podman Quadlet + # blue/green deploy on the dedicated Hetzner hosts. + deploy_mode = "ecs" + service_name = "oonifindings" repo = "ooni/backend" branch_name = "master" @@ -1004,6 +1140,12 @@ module "ooniapi_oonifindings_deployer" { ecs_service_name = module.ooniapi_oonifindings.ecs_service_name ecs_cluster_name = module.ooniapi_cluster.cluster_name + + # Pre-wired for the future flip to deploy_mode = "blue_green" + env_vars = {} + secrets = keys(local.ooniapi_deploy_service_secrets.oonifindings) + service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["oonifindings"].arn + deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn } module "ooniapi_oonifindings" { @@ -1043,6 +1185,10 @@ module "ooniapi_oonifindings" { module "ooniapi_ooniauth_deployer" { source = "../../modules/ooniapi_service_deployer" + # Flip to "blue_green" to switch this service to the Podman Quadlet + # blue/green deploy on the dedicated Hetzner hosts. + deploy_mode = "ecs" + service_name = "ooniauth" repo = "ooni/backend" branch_name = "master" @@ -1055,6 +1201,27 @@ module "ooniapi_ooniauth_deployer" { ecs_service_name = module.ooniapi_ooniauth.ecs_service_name ecs_cluster_name = module.ooniapi_cluster.cluster_name + + # Pre-wired for the future flip to deploy_mode = "blue_green" + env_vars = { + AWS_REGION = var.aws_region + EMAIL_SOURCE_ADDRESS = module.ooniapi_user.email_address + SESSION_EXPIRY_DAYS = 2 + LOGIN_EXPIRY_DAYS = 7 + ADMIN_EMAILS = jsonencode([ + "maja@ooni.org", + "arturo@ooni.org", + "mehul@ooni.org", + "norbel@ooni.org", + "maria@ooni.org", + "admin+dev@ooni.org", + "luis@openobservatory.org", + "contact@openobservatory.org" + ]) + } + secrets = keys(local.ooniapi_deploy_service_secrets.ooniauth) + service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["ooniauth"].arn + deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn } module "ooniapi_ooniauth" { @@ -1112,6 +1279,10 @@ module "ooniapi_ooniauth" { module "ooniapi_oonimeasurements_deployer" { source = "../../modules/ooniapi_service_deployer" + # Flip to "blue_green" to switch this service to the Podman Quadlet + # blue/green deploy on the dedicated Hetzner hosts. + deploy_mode = "ecs" + service_name = "oonimeasurements" repo = "ooni/backend" branch_name = "master" @@ -1122,6 +1293,21 @@ module "ooniapi_oonimeasurements_deployer" { codepipeline_bucket = aws_s3_bucket.ooniapi_codepipeline_bucket.bucket + # Pre-wired for the future flip to deploy_mode = "blue_green" + env_vars = { + # it has to be a json-compliant array + OTHER_COLLECTORS = jsonencode([for h in local.fastpath_hosts : "http://${h}:8475"]) + BASE_URL = "https://api.${local.environment}.ooni.io" + S3_BUCKET_NAME = "ooni-data-eu-fra-test" + VALKEY_URL = local.ooniapi_valkey_url + RATE_LIMITS = "10/minute;400000/day;200000/7day" + RATE_LIMITS_WHITELISTED_IPADDRS = jsonencode(["5.9.112.244"]) + RATE_LIMITS_UNMETERED_PAGES = jsonencode(["/metrics", "/health"]) + } + secrets = keys(local.ooniapi_deploy_service_secrets.oonimeasurements) + service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["oonimeasurements"].arn + deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn + ecs_service_name = module.ooniapi_oonimeasurements.ecs_service_name ecs_cluster_name = module.oonitier1plus_cluster.cluster_name } diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf index 2fb42ce7..dfa6de02 100644 --- a/tf/environments/prod/main.tf +++ b/tf/environments/prod/main.tf @@ -279,6 +279,95 @@ resource "aws_secretsmanager_secret_version" "oonipg_url" { ) } +### Blue/green deploy secrets (Podman Quadlet on dedicated Hetzner hosts) +# +# These mirror the values already passed as `task_secrets` to the ECS task +# definitions above, but consolidated into one JSON blob per service so the +# "blue_green" deploy_mode CodeBuild job can fetch them in a single +# secretsmanager:GetSecretValue call and push each key as a Podman secret. +# Values are pulled from the same underlying data sources used by the ECS +# `task_secrets` maps, so both deploy paths stay in sync automatically. + +data "aws_secretsmanager_secret_version" "ooniapi_user_access_key_id" { + secret_id = module.ooniapi_user.aws_access_key_id_arn +} + +data "aws_secretsmanager_secret_version" "ooniapi_user_secret_access_key" { + secret_id = module.ooniapi_user.aws_secret_access_key_arn +} + +locals { + ooniapi_deploy_service_secrets = { + reverseproxy = { + PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value + } + ooniprobe = { + POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value + JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value + PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_write_url.value + ANONC_SECRET_KEY = data.aws_ssm_parameter.anonc_secret_key.value + } + oonirun = { + POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value + JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value + PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_readonly_url.value + } + oonifindings = { + POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value + JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value + PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value + } + ooniauth = { + POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value + JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value + PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value + ACCOUNT_ID_HASHING_KEY = data.aws_ssm_parameter.account_id_hashing_key.value + AWS_SECRET_ACCESS_KEY = data.aws_secretsmanager_secret_version.ooniapi_user_secret_access_key.secret_string + AWS_ACCESS_KEY_ID = data.aws_secretsmanager_secret_version.ooniapi_user_access_key_id.secret_string + } + oonimeasurements = { + POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value + JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value + PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_oonimeasurements_url.value + ACCOUNT_ID_HASHING_KEY = data.aws_ssm_parameter.account_id_hashing_key.value + } + } +} + +resource "aws_secretsmanager_secret" "ooniapi_deploy_service_secrets" { + for_each = local.ooniapi_deploy_service_secrets + + name = "oonidevops/ooniapi/${each.key}/service_secrets" + tags = local.tags +} + +resource "aws_secretsmanager_secret_version" "ooniapi_deploy_service_secrets" { + for_each = local.ooniapi_deploy_service_secrets + + secret_id = aws_secretsmanager_secret.ooniapi_deploy_service_secrets[each.key].id + secret_string = jsonencode(each.value) +} + +# Shared by every service's blue/green deploy job. The private key itself is +# generated and rotated out-of-band (see the deploy README); Terraform only +# owns the secret container, not its value. +resource "aws_secretsmanager_secret" "ooniapi_deploy_ssh_key" { + name = "oonidevops/ooniapi/deploy_ssh_key" + tags = local.tags +} + +resource "aws_secretsmanager_secret_version" "ooniapi_deploy_ssh_key" { + secret_id = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.id + secret_string = "REPLACE_ME: populate out-of-band with the \"deploy\" user's SSH private key" + + lifecycle { + ignore_changes = [secret_string] + } +} + module "geoip_bucket" { source = "../../modules/s3_bucket" @@ -433,6 +522,10 @@ module "ooni_th_droplet" { module "ooniapi_reverseproxy_deployer" { source = "../../modules/ooniapi_service_deployer" + # Flip to "blue_green" to switch this service to the Podman Quadlet + # blue/green deploy on the dedicated Hetzner hosts. + deploy_mode = "ecs" + service_name = "reverseproxy" repo = "ooni/backend" branch_name = "master" @@ -445,6 +538,14 @@ module "ooniapi_reverseproxy_deployer" { ecs_service_name = module.ooniapi_reverseproxy.ecs_service_name ecs_cluster_name = module.ooniapi_cluster.cluster_name + + # Pre-wired for the future flip to deploy_mode = "blue_green" + env_vars = { + TARGET_URL = "https://backend-fsn.ooni.org/" + } + secrets = keys(local.ooniapi_deploy_service_secrets.reverseproxy) + service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["reverseproxy"].arn + deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn } module "ooniapi_reverseproxy" { @@ -857,6 +958,10 @@ EOF module "ooniapi_ooniprobe_deployer" { source = "../../modules/ooniapi_service_deployer" + # Flip to "blue_green" to switch this service to the Podman Quadlet + # blue/green deploy on the dedicated Hetzner hosts. + deploy_mode = "ecs" + service_name = "ooniprobe" repo = "ooni/backend" branch_name = "master" @@ -869,6 +974,23 @@ module "ooniapi_ooniprobe_deployer" { ecs_service_name = module.ooniapi_ooniprobe.ecs_service_name ecs_cluster_name = module.ooniapi_cluster.cluster_name + + # Pre-wired for the future flip to deploy_mode = "blue_green" + env_vars = { + # hardcoded IP for fastpath2.prod.prod.ooni.io + FASTPATH_URL = "http://10.0.0.32:8472" + FASTPATH_URLS = jsonencode([for h in local.fastpath_hosts : "http://${h}:8472"]) + FAILED_REPORTS_BUCKET = aws_s3_bucket.ooniprobe_failed_reports.bucket + COLLECTOR_ID = 4 # be sure this is different from dev + CONFIG_BUCKET = aws_s3_bucket.ooni_private_config_bucket.bucket + TOR_TARGETS = "tor_targets.json" + PSIPHON_CONFIG = "psiphon_config.json" + ANONC_MANIFEST_BUCKET = aws_s3_bucket.anoncred_manifests.bucket + ANONC_MANIFEST_FILE = "manifest.json" + } + secrets = keys(local.ooniapi_deploy_service_secrets.ooniprobe) + service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["ooniprobe"].arn + deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn } module "ooniapi_ooniprobe" { @@ -1027,6 +1149,10 @@ module "fastpath_builder" { module "ooniapi_oonirun_deployer" { source = "../../modules/ooniapi_service_deployer" + # Flip to "blue_green" to switch this service to the Podman Quadlet + # blue/green deploy on the dedicated Hetzner hosts. + deploy_mode = "ecs" + service_name = "oonirun" repo = "ooni/backend" branch_name = "master" @@ -1039,6 +1165,12 @@ module "ooniapi_oonirun_deployer" { ecs_service_name = module.ooniapi_oonirun.ecs_service_name ecs_cluster_name = module.ooniapi_cluster.cluster_name + + # Pre-wired for the future flip to deploy_mode = "blue_green" + env_vars = {} + secrets = keys(local.ooniapi_deploy_service_secrets.oonirun) + service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["oonirun"].arn + deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn } module "ooniapi_oonirun" { @@ -1079,6 +1211,10 @@ module "ooniapi_oonirun" { module "ooniapi_oonifindings_deployer" { source = "../../modules/ooniapi_service_deployer" + # Flip to "blue_green" to switch this service to the Podman Quadlet + # blue/green deploy on the dedicated Hetzner hosts. + deploy_mode = "ecs" + service_name = "oonifindings" repo = "ooni/backend" branch_name = "master" @@ -1091,6 +1227,12 @@ module "ooniapi_oonifindings_deployer" { ecs_service_name = module.ooniapi_oonifindings.ecs_service_name ecs_cluster_name = module.ooniapi_cluster.cluster_name + + # Pre-wired for the future flip to deploy_mode = "blue_green" + env_vars = {} + secrets = keys(local.ooniapi_deploy_service_secrets.oonifindings) + service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["oonifindings"].arn + deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn } module "ooniapi_oonifindings" { @@ -1131,6 +1273,10 @@ module "ooniapi_oonifindings" { module "ooniapi_ooniauth_deployer" { source = "../../modules/ooniapi_service_deployer" + # Flip to "blue_green" to switch this service to the Podman Quadlet + # blue/green deploy on the dedicated Hetzner hosts. + deploy_mode = "ecs" + service_name = "ooniauth" repo = "ooni/backend" branch_name = "master" @@ -1143,6 +1289,27 @@ module "ooniapi_ooniauth_deployer" { ecs_service_name = module.ooniapi_ooniauth.ecs_service_name ecs_cluster_name = module.ooniapi_cluster.cluster_name + + # Pre-wired for the future flip to deploy_mode = "blue_green" + env_vars = { + AWS_REGION = var.aws_region + EMAIL_SOURCE_ADDRESS = module.ooniapi_user.email_address + SESSION_EXPIRY_DAYS = 2 + LOGIN_EXPIRY_DAYS = 7 + ADMIN_EMAILS = jsonencode([ + "maja@ooni.org", + "arturo@ooni.org", + "mehul@ooni.org", + "norbel@ooni.org", + "maria@ooni.org", + "admin+dev@ooni.org", + "luis@openobservatory.org", + "contact@openobservatory.org" + ]) + } + secrets = keys(local.ooniapi_deploy_service_secrets.ooniauth) + service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["ooniauth"].arn + deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn } module "ooniapi_ooniauth" { @@ -1202,6 +1369,10 @@ module "ooniapi_ooniauth" { module "ooniapi_oonimeasurements_deployer" { source = "../../modules/ooniapi_service_deployer" + # Flip to "blue_green" to switch this service to the Podman Quadlet + # blue/green deploy on the dedicated Hetzner hosts. + deploy_mode = "ecs" + service_name = "oonimeasurements" repo = "ooni/backend" branch_name = "master" @@ -1215,6 +1386,21 @@ module "ooniapi_oonimeasurements_deployer" { ecs_service_name = module.ooniapi_oonimeasurements.ecs_service_name ecs_cluster_name = module.oonitier1plus_cluster.cluster_name # ecs_cluster_name = module.ooniapi_cluster.cluster_name + + # Pre-wired for the future flip to deploy_mode = "blue_green" + env_vars = { + # it has to be a json-compliant array + OTHER_COLLECTORS = jsonencode([for h in local.fastpath_hosts : "http://${h}:8475"]) + BASE_URL = "https://api.ooni.io" + S3_BUCKET_NAME = "ooni-data-eu-fra" + VALKEY_URL = local.ooniapi_valkey_url + RATE_LIMITS = "4000/hour;400000/day;200000/7day" + RATE_LIMITS_WHITELISTED_IPADDRS = jsonencode(["5.9.112.244"]) + RATE_LIMITS_UNMETERED_PAGES = jsonencode(["/metrics", "/health"]) + } + secrets = keys(local.ooniapi_deploy_service_secrets.oonimeasurements) + service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["oonimeasurements"].arn + deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn } module "ooniapi_oonimeasurements" { From b09d13783ce5e8db7e8c550d7dc86ca075484a55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Fri, 10 Jul 2026 21:52:25 +0200 Subject: [PATCH 03/24] Add ansible setup for ooniapi-gateway --- ansible/deploy-ooniapi-gateway.yml | 7 + .../roles/ooniapi_gateway/defaults/main.yml | 63 +++++++ .../roles/ooniapi_gateway/handlers/main.yml | 9 + ansible/roles/ooniapi_gateway/tasks/main.yml | 119 +++++++++++++ .../ooniapi_gateway/templates/gateway.conf.j2 | 167 ++++++++++++++++++ .../templates/sudoers-ooniapi-deploy.j2 | 23 +++ .../templates/buildspec_deploy.yml | 2 +- 7 files changed, 389 insertions(+), 1 deletion(-) create mode 100644 ansible/deploy-ooniapi-gateway.yml create mode 100644 ansible/roles/ooniapi_gateway/defaults/main.yml create mode 100644 ansible/roles/ooniapi_gateway/handlers/main.yml create mode 100644 ansible/roles/ooniapi_gateway/tasks/main.yml create mode 100644 ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 create mode 100644 ansible/roles/ooniapi_gateway/templates/sudoers-ooniapi-deploy.j2 diff --git a/ansible/deploy-ooniapi-gateway.yml b/ansible/deploy-ooniapi-gateway.yml new file mode 100644 index 00000000..f65c03a9 --- /dev/null +++ b/ansible/deploy-ooniapi-gateway.yml @@ -0,0 +1,7 @@ +--- +- name: Bootstrap the tier0 blue/green gateway on dedicated backend hosts + hosts: backend-fsn.ooni.org:backend-hel.ooni.org + become: true + roles: + - role: ooniapi_gateway + tags: ooniapi_gateway diff --git a/ansible/roles/ooniapi_gateway/defaults/main.yml b/ansible/roles/ooniapi_gateway/defaults/main.yml new file mode 100644 index 00000000..57ef60dd --- /dev/null +++ b/ansible/roles/ooniapi_gateway/defaults/main.yml @@ -0,0 +1,63 @@ +--- +# Domain the gateway vhost answers on for path-based routing. Mirrors the +# ALB's `ooniapi_frontend_main_domain_name` in +# tf/modules/ooniapi_frontend/main.tf (e.g. "api.prod.ooni.io"). +# No default: set explicitly per host in host_vars. +ooniapi_gateway_primary_domain: "" + +# Any additional hostnames that should hit the same path-routed vhost. +# Mirrors extra entries in `ooniapi_frontend_alternative_domains` that +# aren't handled elsewhere (e.g. "api.ooni.org" in prod). +ooniapi_gateway_extra_server_names: [] + +# Suffix used for the ALB's secondary, host-header-only routing rules, e.g. +# "oonirun.prod.ooni.io". Mirrors `direct_domain_suffix` in +# tf/modules/ooniapi_frontend/main.tf ("${stage}.ooni.io"). +# No default: set explicitly per host in host_vars. +ooniapi_gateway_direct_domain_suffix: "" + +# Name of the dehydrated cert (the directory under +# ooniapi_gateway_cert_path) that covers ooniapi_gateway_primary_domain, +# every ".{{ ooniapi_gateway_direct_domain_suffix }}" name, and +# ooniapi_gateway_extra_server_names as SANs. +# +# This role does NOT manage that cert -- issuing it means extending (or +# adding to) the `dehydrated` role's ssl_domains on this host, which in turn +# requires DNS for every one of those names to already resolve here. That's +# a DNS-cutover-time step, not a day-1 bootstrap step, so it's deliberately +# left manual. See the role README notes. +ooniapi_gateway_cert_name: "{{ ooniapi_gateway_primary_domain }}" +ooniapi_gateway_cert_path: /var/lib/dehydrated/certs/ + +# Flip to true only once ooniapi_gateway_cert_name actually exists on disk. +# Until then the vhost file is rendered but not installed, so this role is +# safe to run ahead of the real DNS cutover. +ooniapi_gateway_cert_ready: false + +# Services fronted by this gateway, and the two host ports their blue/green +# slots are bound to. host_port_a/host_port_b must match the values passed +# to that service's ooniapi_service_deployer module invocation in Terraform. +# +# Example: +# ooniapi_gateway_services: +# - name: reverseproxy +# host_port_a: 18001 +# host_port_b: 18002 +# - name: oonirun +# host_port_a: 18011 +# host_port_b: 18012 +ooniapi_gateway_services: [] + +# Cross-cloud passthrough for tier0 services not yet migrated off AWS +# (currently just testlists). Set to its public hostname to proxy those +# paths there until testlists itself is migrated; leave empty to omit +# those routes entirely. +ooniapi_gateway_testlists_upstream: "" + +# Podman network shared by every service's containers on this host. +ooniapi_gateway_network_name: ooniapi + +# SSH user the deploy CodeBuild job connects as. Must match +# `deploy_ssh_user` (default "deploy") in the ooniapi_service_deployer +# Terraform module. +ooniapi_gateway_deploy_user: deploy diff --git a/ansible/roles/ooniapi_gateway/handlers/main.yml b/ansible/roles/ooniapi_gateway/handlers/main.yml new file mode 100644 index 00000000..0fb3a40e --- /dev/null +++ b/ansible/roles/ooniapi_gateway/handlers/main.yml @@ -0,0 +1,9 @@ +--- +- name: test ooniapi gateway nginx config + ansible.builtin.command: /usr/sbin/nginx -t -c /etc/nginx/nginx.conf + listen: reload ooniapi gateway nginx + +- name: reload ooniapi gateway nginx + ansible.builtin.service: + name: nginx + state: reloaded diff --git a/ansible/roles/ooniapi_gateway/tasks/main.yml b/ansible/roles/ooniapi_gateway/tasks/main.yml new file mode 100644 index 00000000..94550e86 --- /dev/null +++ b/ansible/roles/ooniapi_gateway/tasks/main.yml @@ -0,0 +1,119 @@ +--- +# One-time (and idempotent-on-rerun) bootstrap for a host that will run +# tier0 services via Podman Quadlet blue/green deploys, plus the nginx +# gateway vhost that fronts them. See the Terraform side of this in +# tf/modules/ooniapi_service_deployer (deploy_mode = "blue_green") and its +# templates/buildspec_deploy.yml, which is what actually performs each +# deploy -- this role only prepares the host for that job to run against. +# +# Runs as root (become: true at the play level). Podman state (networks, +# secrets) is created in root's namespace to match the root-owned systemd +# Quadlet units under /etc/containers/systemd/. + +- name: Ensure per-service state directories exist + ansible.builtin.file: + path: "/etc/ooniapi/{{ item.name }}" + state: directory + owner: root + group: root + mode: "0755" + loop: "{{ ooniapi_gateway_services }}" + loop_control: + label: "{{ item.name }}" + +- name: Check whether each service already has an active_slot marker + ansible.builtin.stat: + path: "/etc/ooniapi/{{ item.name }}/active_slot" + loop: "{{ ooniapi_gateway_services }}" + loop_control: + label: "{{ item.name }}" + register: ooniapi_gateway_active_slot_stat + +- name: Seed active_slot with "a" for services deploying for the first time + ansible.builtin.copy: + dest: "/etc/ooniapi/{{ item.item.name }}/active_slot" + content: "a" + owner: root + group: root + mode: "0644" + loop: "{{ ooniapi_gateway_active_slot_stat.results }}" + loop_control: + label: "{{ item.item.name }}" + when: not item.stat.exists + +- name: Check whether each service already has an nginx upstream conf + ansible.builtin.stat: + path: "/etc/nginx/conf.d/{{ item.name }}-upstream.conf" + loop: "{{ ooniapi_gateway_services }}" + loop_control: + label: "{{ item.name }}" + register: ooniapi_gateway_upstream_stat + +# Seeded with slot "a" up / "b" down, matching active_slot's "a" default +# above, so the very first `nginx -t` (run when installing the gateway +# vhost below) succeeds before that service's first real deploy has run. +- name: Seed a placeholder upstream conf for services deploying for the first time + ansible.builtin.copy: + dest: "/etc/nginx/conf.d/{{ item.item.name }}-upstream.conf" + content: | + upstream {{ item.item.name }} { + server 127.0.0.1:{{ item.item.host_port_a }}; + server 127.0.0.1:{{ item.item.host_port_b }} down; + } + owner: root + group: root + mode: "0644" + loop: "{{ ooniapi_gateway_upstream_stat.results }}" + loop_control: + label: "{{ item.item.name }}" + when: not item.stat.exists + notify: reload ooniapi gateway nginx + +- name: Check whether the podman network already exists + ansible.builtin.command: podman network exists {{ ooniapi_gateway_network_name }} + register: ooniapi_gateway_network_check + changed_when: false + failed_when: false + +- name: Create the shared podman network + ansible.builtin.command: podman network create {{ ooniapi_gateway_network_name }} + when: ooniapi_gateway_network_check.rc != 0 + +- name: Ensure sudoers.d directory exists + ansible.builtin.file: + path: /etc/sudoers.d + state: directory + owner: root + group: root + +- name: Install scoped sudoers rule for the deploy user + ansible.builtin.template: + src: sudoers-ooniapi-deploy.j2 + dest: /etc/sudoers.d/90-ooniapi-deploy + owner: root + group: root + mode: "0440" + validate: "visudo -cf %s" + +- name: Render the gateway vhost + ansible.builtin.template: + src: gateway.conf.j2 + dest: /etc/nginx/conf.d/ooniapi-gateway.conf.pending + owner: root + group: root + mode: "0644" + +# Split into "render" + "install" so a bad template is visible +# (.conf.pending) without ever being loaded by nginx, and so the vhost is +# never installed at all until ooniapi_gateway_cert_ready is true -- see +# defaults/main.yml for why that's a separate, deliberate step. +- name: Install the gateway vhost now that its cert is ready + ansible.builtin.copy: + remote_src: true + src: /etc/nginx/conf.d/ooniapi-gateway.conf.pending + dest: /etc/nginx/conf.d/ooniapi-gateway.conf + owner: root + group: root + mode: "0644" + when: ooniapi_gateway_cert_ready + notify: reload ooniapi gateway nginx diff --git a/ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 b/ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 new file mode 100644 index 00000000..e87e46ef --- /dev/null +++ b/ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 @@ -0,0 +1,167 @@ +# Managed by ansible - roles/ooniapi_gateway/templates/gateway.conf.j2 +# +# Single "gateway" vhost for the OONI tier0 API on this dedicated host, +# mirroring (1:1, by design) the AWS ALB listener rules in +# tf/modules/ooniapi_frontend/main.tf. Only rules for services actually +# running blue/green here ({{ ooniapi_gateway_services | map(attribute='name') | join(', ') }}) +# are handled; everything unmatched falls through to "reverseproxy", exactly +# like the ALB's default_action. +# +# Each service's own `-upstream.conf` (the `upstream { +# ... }` block referenced below via proxy_pass) is dropped into this same +# conf.d directory by that service's blue/green deploy job, not by this +# role -- see ooniapi_service_deployer's buildspec_deploy.yml. A service's +# location block here will fail nginx -t until that service has deployed at +# least once. + +{% set service_names = ooniapi_gateway_services | map(attribute='name') | list %} + +# anonymize ipaddr (same scheme as the legacy ooni-api vhost) +map $remote_addr $ooniapi_gateway_remote_addr_anon { + ~(?P\d+\.\d+\.\d+)\. $ip.0; + ~(?P[^:]+:[^:]+): $ip::; + default 0.0.0.0; +} + +log_format ooniapi_gateway_fmt '$ooniapi_gateway_remote_addr_anon [$time_local] ' + '"$request" $status $body_bytes_sent rt:$request_time "$http_referer" "$http_user_agent"'; + +server { + listen 443 ssl http2; + listen [::]:443 ssl http2; + server_name {{ ([ooniapi_gateway_primary_domain] + ooniapi_gateway_extra_server_names) | join(' ') }}; + + access_log syslog:server=unix:/dev/log,tag=ooniapi-gateway,severity=info ooniapi_gateway_fmt; + error_log syslog:server=unix:/dev/log,tag=ooniapi-gateway,severity=info; + + client_max_body_size 200M; # for measurement POST, matches the legacy API vhost + + ssl_certificate {{ ooniapi_gateway_cert_path }}{{ ooniapi_gateway_cert_name }}/fullchain.pem; + ssl_certificate_key {{ ooniapi_gateway_cert_path }}{{ ooniapi_gateway_cert_name }}/privkey.pem; + ssl_trusted_certificate {{ ooniapi_gateway_cert_path }}{{ ooniapi_gateway_cert_name }}/chain.pem; + + # https://ssl-config.mozilla.org/#server=nginx&config=intermediate -- kept + # inline (rather than shared include) so this role has no dependency on + # how nginx itself got installed on this host. + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384; + ssl_prefer_server_ciphers off; + ssl_session_timeout 5m; + ssl_session_cache shared:MozSSL:30m; + ssl_session_tickets off; + ssl_stapling on; + ssl_stapling_verify on; + + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; + add_header X-Frame-Options DENY always; + add_header X-Content-Type-Options nosniff always; + + proxy_http_version 1.1; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 900; + +{% if 'ooniauth' in service_names %} + ## --- ooniauth --- (ALB priority 108) + location ^~ /api/v2/ooniauth/ { proxy_pass http://ooniauth; } + location = /api/v1/user_register { proxy_pass http://ooniauth; } + location = /api/v1/user_login { proxy_pass http://ooniauth; } + location = /api/v1/user_refresh_token { proxy_pass http://ooniauth; } + location = /api/_/account_metadata { proxy_pass http://ooniauth; } +{% endif %} + +{% if 'oonirun' in service_names %} + ## --- oonirun --- (ALB priority 110) + location ^~ /api/v2/oonirun/ { proxy_pass http://oonirun; } +{% endif %} + +{% if 'ooniprobe' in service_names %} + ## --- ooniprobe --- (ALB priorities 120-123) + location ^~ /api/v2/ooniprobe/ { proxy_pass http://ooniprobe; } + location = /api/v1/login { proxy_pass http://ooniprobe; } + location = /api/v1/register { proxy_pass http://ooniprobe; } + location ^~ /api/v1/update/ { proxy_pass http://ooniprobe; } + location ^~ /api/v1/check-in { proxy_pass http://ooniprobe; } + location ^~ /api/v1/test-helpers { proxy_pass http://ooniprobe; } + location = /api/v1/test-list/urls { proxy_pass http://ooniprobe; } + location ^~ /report { proxy_pass http://ooniprobe; } + location = /api/_/show_countries_prioritization { proxy_pass http://ooniprobe; } + location = /api/_/debug_prioritization { proxy_pass http://ooniprobe; } + location ^~ /api/v1/manifest { proxy_pass http://ooniprobe; } + location ^~ /api/v1/sign_credential { proxy_pass http://ooniprobe; } + location ^~ /api/v1/submit_measurement { proxy_pass http://ooniprobe; } + location ^~ /bouncer/net-tests { proxy_pass http://ooniprobe; } + location ^~ /api/v1/geolookup { proxy_pass http://ooniprobe; } + location ^~ /api/v1/collectors { proxy_pass http://ooniprobe; } + location = /api/v1/test-list/tor-targets { proxy_pass http://ooniprobe; } + location = /api/v1/test-list/psiphon-config { proxy_pass http://ooniprobe; } +{% endif %} + +{% if 'oonifindings' in service_names %} + ## --- oonifindings --- (ALB priority 130) + location ^~ /api/v1/incidents/ { proxy_pass http://oonifindings; } +{% endif %} + +{% if 'oonimeasurements' in service_names %} + ## --- oonimeasurements --- (ALB priorities 140-143) + location ^~ /api/v1/measurements/ { proxy_pass http://oonimeasurements; } + location = /api/v1/raw_measurement { proxy_pass http://oonimeasurements; } + location = /api/v1/measurement_meta { proxy_pass http://oonimeasurements; } + location = /api/v1/measurements { proxy_pass http://oonimeasurements; } + location = /api/v1/torsf_stats { proxy_pass http://oonimeasurements; } + location = /api/v1/aggregation { proxy_pass http://oonimeasurements; } + location ^~ /api/v1/aggregation/ { proxy_pass http://oonimeasurements; } + location = /api/v1/observations { proxy_pass http://oonimeasurements; } + location = /api/v1/analysis { proxy_pass http://oonimeasurements; } + location = /api/v1/detector/changepoints { proxy_pass http://oonimeasurements; } +{% endif %} + +{% if ooniapi_gateway_testlists_upstream %} + ## --- testlists (ALB priority 144; not yet migrated off AWS, proxied + ## cross-cloud until it is) --- + location ^~ /api/_/url-submission/test-list/ { proxy_pass https://{{ ooniapi_gateway_testlists_upstream }}; } + location = /api/_/url-priorities/list { proxy_pass https://{{ ooniapi_gateway_testlists_upstream }}; } + location = /api/_/url-priorities/update { proxy_pass https://{{ ooniapi_gateway_testlists_upstream }}; } + location = /api/v1/url-submission/submit { proxy_pass https://{{ ooniapi_gateway_testlists_upstream }}; } + location = /api/v1/url-submission/update-url { proxy_pass https://{{ ooniapi_gateway_testlists_upstream }}; } +{% endif %} + + ## --- default: everything else goes to reverseproxy, same as the ALB's + ## default_action. Covers legacy/unmigrated paths (oonith is NOT among + ## them -- it's routed by host_header on *.th.ooni.org, a hostname this + ## vhost doesn't answer for, so it never reaches here either way). --- + location / { + proxy_pass http://reverseproxy; + } +} + +{% for service in service_names %} +{% if service != 'reverseproxy' %} +server { + listen 443 ssl http2; + listen [::]:443 ssl http2; + server_name {{ service }}.{{ ooniapi_gateway_direct_domain_suffix }}; + + access_log syslog:server=unix:/dev/log,tag=ooniapi-gateway,severity=info ooniapi_gateway_fmt; + error_log syslog:server=unix:/dev/log,tag=ooniapi-gateway,severity=info; + + ssl_certificate {{ ooniapi_gateway_cert_path }}{{ ooniapi_gateway_cert_name }}/fullchain.pem; + ssl_certificate_key {{ ooniapi_gateway_cert_path }}{{ ooniapi_gateway_cert_name }}/privkey.pem; + ssl_trusted_certificate {{ ooniapi_gateway_cert_path }}{{ ooniapi_gateway_cert_name }}/chain.pem; + ssl_protocols TLSv1.2 TLSv1.3; + + proxy_http_version 1.1; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 900; + + location / { + proxy_pass http://{{ service }}; + } +} +{% endif %} +{% endfor %} diff --git a/ansible/roles/ooniapi_gateway/templates/sudoers-ooniapi-deploy.j2 b/ansible/roles/ooniapi_gateway/templates/sudoers-ooniapi-deploy.j2 new file mode 100644 index 00000000..d78a8d27 --- /dev/null +++ b/ansible/roles/ooniapi_gateway/templates/sudoers-ooniapi-deploy.j2 @@ -0,0 +1,23 @@ +# Managed by ansible - roles/ooniapi_gateway/templates/sudoers-ooniapi-deploy.j2 +# +# Scoped to exactly the remote commands issued by +# tf/modules/ooniapi_service_deployer/templates/buildspec_deploy.yml over +# SSH as {{ ooniapi_gateway_deploy_user }}. Intentionally narrower than the +# blanket "NOPASSWD:ALL" pattern used for the deploy user in other roles +# (e.g. roles/anonc) -- this user is driven by an automated CI job over the +# network, not a human operator. +# +# Caveat: sudoers glob matching (fnmatch without FNM_PATHNAME) lets "*" +# match "/". The mv sources below are always files this same job just +# scp'd into /tmp itself, so this is a theoretical rather than practical +# widening of scope, but worth knowing if you're auditing this file. + +Cmnd_Alias OONIAPI_DEPLOY_MV_UNIT = /usr/bin/mv /tmp/*.container /etc/containers/systemd/*.container +Cmnd_Alias OONIAPI_DEPLOY_MV_UPSTREAM = /usr/bin/mv /tmp/*-upstream.conf /etc/nginx/conf.d/*-upstream.conf +Cmnd_Alias OONIAPI_DEPLOY_SYSTEMCTL = /usr/bin/systemctl daemon-reload, /usr/bin/systemctl restart *-a.service, /usr/bin/systemctl restart *-b.service, /usr/bin/systemctl reload nginx +Cmnd_Alias OONIAPI_DEPLOY_NGINX_TEST = /usr/sbin/nginx -t +Cmnd_Alias OONIAPI_DEPLOY_MARKER = /usr/bin/tee /etc/ooniapi/*/active_slot +Cmnd_Alias OONIAPI_DEPLOY_PODMAN_SECRET = /usr/bin/podman secret create --replace * - + +{{ ooniapi_gateway_deploy_user }} ALL=(root) NOPASSWD: OONIAPI_DEPLOY_MV_UNIT, OONIAPI_DEPLOY_MV_UPSTREAM, OONIAPI_DEPLOY_SYSTEMCTL, OONIAPI_DEPLOY_NGINX_TEST, OONIAPI_DEPLOY_MARKER +{{ ooniapi_gateway_deploy_user }} ALL=(root) NOPASSWD: OONIAPI_DEPLOY_PODMAN_SECRET diff --git a/tf/modules/ooniapi_service_deployer/templates/buildspec_deploy.yml b/tf/modules/ooniapi_service_deployer/templates/buildspec_deploy.yml index 596aad30..1ea3968e 100644 --- a/tf/modules/ooniapi_service_deployer/templates/buildspec_deploy.yml +++ b/tf/modules/ooniapi_service_deployer/templates/buildspec_deploy.yml @@ -49,7 +49,7 @@ phases: # stdin, never interpolated into the remote command string) for KEY in $(jq -r 'keys[]' /tmp/service_secrets.json); do VALUE=$(jq -r --arg k "$KEY" '.[$k]' /tmp/service_secrets.json) - printf '%s' "$VALUE" | ssh $SSH_OPTS "${DEPLOY_SSH_USER}@${HOST}" "podman secret create --replace ${KEY} -" + printf '%s' "$VALUE" | ssh $SSH_OPTS "${DEPLOY_SSH_USER}@${HOST}" "sudo podman secret create --replace ${KEY} -" done # d. reload systemd and (re)start only the target slot From e11b07185b0e446805c9f47401cac30fe5291ba8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Thu, 13 Aug 2026 18:09:24 +0200 Subject: [PATCH 04/24] Switch blue/green deployer from Podman Quadlet to Docker Compose - Install docker-ce from Docker's official apt repo (distro-detected via ansible_distribution_release, not hardcoded), replacing Debian's docker.io/podman with containerd's nftables firewall backend enabled. Verify the downloaded apt signing key's fingerprint before trusting it. - Rewrite the deploy stage as a single dependency-free deploy.py (stdlib only, shells out to ssh/scp/aws/docker) instead of inline buildspec bash. - Render one docker-compose.yaml per slot instead of a Quadlet unit; slots are supervised by docker's own restart policy, no systemd unit needed. - Move service secrets from a shared env file to Compose's native file-based secrets, so each is mounted read-only at /run/secrets/ instead of ending up in the container's environment. - Stage files scp'd to target hosts in a private, deploy-user-owned directory (chmod 600 + scp -p) instead of world-readable /tmp, closing the window where they'd be briefly readable by other local users. --- .../roles/ooniapi_gateway/defaults/main.yml | 7 +- .../roles/ooniapi_gateway/handlers/main.yml | 5 + ansible/roles/ooniapi_gateway/tasks/main.yml | 128 ++++++++++++- .../templates/sudoers-ooniapi-deploy.j2 | 32 ++-- tf/environments/dev/main.tf | 61 +++--- tf/environments/prod/main.tf | 47 ++--- .../ooniapi_service_deployer/files/deploy.py | 177 ++++++++++++++++++ tf/modules/ooniapi_service_deployer/main.tf | 36 ++-- .../templates/buildspec_deploy.yml | 86 +-------- .../templates/compose.yaml.tftpl | 34 ++++ .../templates/quadlet.container.tftpl | 23 --- .../ooniapi_service_deployer/variables.tf | 24 +-- 12 files changed, 447 insertions(+), 213 deletions(-) create mode 100644 tf/modules/ooniapi_service_deployer/files/deploy.py create mode 100644 tf/modules/ooniapi_service_deployer/templates/compose.yaml.tftpl delete mode 100644 tf/modules/ooniapi_service_deployer/templates/quadlet.container.tftpl diff --git a/ansible/roles/ooniapi_gateway/defaults/main.yml b/ansible/roles/ooniapi_gateway/defaults/main.yml index 57ef60dd..0b28f33b 100644 --- a/ansible/roles/ooniapi_gateway/defaults/main.yml +++ b/ansible/roles/ooniapi_gateway/defaults/main.yml @@ -54,10 +54,15 @@ ooniapi_gateway_services: [] # those routes entirely. ooniapi_gateway_testlists_upstream: "" -# Podman network shared by every service's containers on this host. +# Docker network shared by every service's containers on this host. ooniapi_gateway_network_name: ooniapi # SSH user the deploy CodeBuild job connects as. Must match # `deploy_ssh_user` (default "deploy") in the ooniapi_service_deployer # Terraform module. ooniapi_gateway_deploy_user: deploy + +# Private directory the deploy job scp's files into before they're +# moved/installed into place. Must match STAGING_DIR in +# tf/modules/ooniapi_service_deployer/files/deploy.py. +ooniapi_gateway_staging_dir: /var/lib/ooniapi/deploy-staging diff --git a/ansible/roles/ooniapi_gateway/handlers/main.yml b/ansible/roles/ooniapi_gateway/handlers/main.yml index 0fb3a40e..9d61904d 100644 --- a/ansible/roles/ooniapi_gateway/handlers/main.yml +++ b/ansible/roles/ooniapi_gateway/handlers/main.yml @@ -7,3 +7,8 @@ ansible.builtin.service: name: nginx state: reloaded + +- name: restart docker + ansible.builtin.systemd_service: + name: docker + state: restarted diff --git a/ansible/roles/ooniapi_gateway/tasks/main.yml b/ansible/roles/ooniapi_gateway/tasks/main.yml index 94550e86..020790d3 100644 --- a/ansible/roles/ooniapi_gateway/tasks/main.yml +++ b/ansible/roles/ooniapi_gateway/tasks/main.yml @@ -1,14 +1,13 @@ --- # One-time (and idempotent-on-rerun) bootstrap for a host that will run -# tier0 services via Podman Quadlet blue/green deploys, plus the nginx +# tier0 services via Docker Compose blue/green deploys, plus the nginx # gateway vhost that fronts them. See the Terraform side of this in # tf/modules/ooniapi_service_deployer (deploy_mode = "blue_green") and its -# templates/buildspec_deploy.yml, which is what actually performs each -# deploy -- this role only prepares the host for that job to run against. +# files/deploy.py, which is what actually performs each deploy -- this role +# only prepares the host for that job to run against. # -# Runs as root (become: true at the play level). Podman state (networks, -# secrets) is created in root's namespace to match the root-owned systemd -# Quadlet units under /etc/containers/systemd/. +# Runs as root (become: true at the play level). Each deploy runs `docker +# compose up` as root (via the scoped sudoers rule below), matching this. - name: Ensure per-service state directories exist ansible.builtin.file: @@ -21,6 +20,18 @@ loop_control: label: "{{ item.name }}" +# Files land here (not shared /tmp) before being moved/installed into +# place, so the per-secret files scp'd in on every deploy are never briefly +# world-readable on their way in -- only the deploy user (and root) can +# even list this directory. +- name: Ensure the deploy user's private staging directory exists + ansible.builtin.file: + path: "{{ ooniapi_gateway_staging_dir }}" + state: directory + owner: "{{ ooniapi_gateway_deploy_user }}" + group: "{{ ooniapi_gateway_deploy_user }}" + mode: "0700" + - name: Check whether each service already has an active_slot marker ansible.builtin.stat: path: "/etc/ooniapi/{{ item.name }}/active_slot" @@ -69,14 +80,111 @@ when: not item.stat.exists notify: reload ooniapi gateway nginx -- name: Check whether the podman network already exists - ansible.builtin.command: podman network exists {{ ooniapi_gateway_network_name }} +# Follows https://docs.docker.com/engine/install/debian/ (the officially +# supported docker-ce packages, rather than Debian's own docker.io/ +# docker-compose-v2) so we get upstream's release cadence. The repo URL is +# built from ansible_distribution_release rather than a hardcoded codename +# (e.g. "trixie") so this keeps working unchanged across Debian releases. +- name: Remove conflicting docker packages + ansible.builtin.apt: + name: + - docker.io + - docker-doc + - docker-compose + - docker-compose-v2 + - podman-docker + - containerd + - runc + state: absent + +- name: Install docker's prerequisites + ansible.builtin.apt: + name: + - ca-certificates + - curl + - gnupg + state: present + update_cache: true + +- name: Ensure the apt keyrings directory exists + ansible.builtin.file: + path: /etc/apt/keyrings + state: directory + mode: "0755" + +- name: Download docker's official apt signing key + ansible.builtin.get_url: + url: "https://download.docker.com/linux/{{ ansible_distribution | lower }}/gpg" + dest: /etc/apt/keyrings/docker.asc + mode: "0644" + +- name: Read the fingerprint of the downloaded key + ansible.builtin.command: gpg --with-colons --show-keys /etc/apt/keyrings/docker.asc + register: ooniapi_gateway_docker_key_check + changed_when: false + +# Pins against docker's known-good key fingerprint (9DC8 5822 9FC7 DD38 +# 854A E2D8 8D81 803C 0EBF CD88) so a compromised/MITM'd download of the key +# itself (which apt would otherwise trust blindly via signed-by=) gets +# caught here instead of silently being trusted for every future `apt +# update`. +- name: Verify the downloaded key's fingerprint matches docker's published one + ansible.builtin.assert: + that: + - ooniapi_gateway_docker_key_check.stdout is + search('^fpr:::::::::9DC858229FC7DD38854AE2D88D81803C0EBFCD88:$', multiline=true) + fail_msg: >- + docker.asc's fingerprint does not match the expected + 9DC858229FC7DD38854AE2D88D81803C0EBFCD88 -- refusing to trust it. + success_msg: "docker.asc's fingerprint matches the expected value." + +- name: Get dpkg architecture + ansible.builtin.command: dpkg --print-architecture + register: ooniapi_gateway_dpkg_arch + changed_when: false + +- name: Add the docker apt repository + ansible.builtin.apt_repository: + repo: >- + deb [arch={{ ooniapi_gateway_dpkg_arch.stdout }} signed-by=/etc/apt/keyrings/docker.asc] + https://download.docker.com/linux/{{ ansible_distribution | lower }} + {{ ansible_distribution_release }} stable + filename: docker + state: present + +- name: Install docker + ansible.builtin.apt: + name: + - docker-ce + - docker-ce-cli + - containerd.io + - docker-compose-plugin + state: present + update_cache: true + +- name: Make docker's containerd use the nftables firewall backend + ansible.builtin.lineinfile: + path: /etc/containerd/config.toml + line: 'firewall-backend="nftables"' + state: present + create: true + mode: "0644" + notify: restart docker + +- name: Ensure docker is running + ansible.builtin.systemd_service: + name: docker + state: started + enabled: true + +- name: Check whether the shared docker network already exists + ansible.builtin.command: docker network inspect {{ ooniapi_gateway_network_name }} register: ooniapi_gateway_network_check changed_when: false failed_when: false -- name: Create the shared podman network - ansible.builtin.command: podman network create {{ ooniapi_gateway_network_name }} +- name: Create the shared docker network + ansible.builtin.command: docker network create {{ ooniapi_gateway_network_name }} when: ooniapi_gateway_network_check.rc != 0 - name: Ensure sudoers.d directory exists diff --git a/ansible/roles/ooniapi_gateway/templates/sudoers-ooniapi-deploy.j2 b/ansible/roles/ooniapi_gateway/templates/sudoers-ooniapi-deploy.j2 index d78a8d27..dacbb55c 100644 --- a/ansible/roles/ooniapi_gateway/templates/sudoers-ooniapi-deploy.j2 +++ b/ansible/roles/ooniapi_gateway/templates/sudoers-ooniapi-deploy.j2 @@ -1,23 +1,29 @@ # Managed by ansible - roles/ooniapi_gateway/templates/sudoers-ooniapi-deploy.j2 # # Scoped to exactly the remote commands issued by -# tf/modules/ooniapi_service_deployer/templates/buildspec_deploy.yml over -# SSH as {{ ooniapi_gateway_deploy_user }}. Intentionally narrower than the -# blanket "NOPASSWD:ALL" pattern used for the deploy user in other roles -# (e.g. roles/anonc) -- this user is driven by an automated CI job over the +# tf/modules/ooniapi_service_deployer/files/deploy.py over SSH as +# {{ ooniapi_gateway_deploy_user }}. Intentionally narrower than the blanket +# "NOPASSWD:ALL" pattern used for the deploy user in other roles (e.g. +# roles/anonc) -- this user is driven by an automated CI job over the # network, not a human operator. # +# Sources below are always files this same job just scp'd into +# {{ ooniapi_gateway_staging_dir }}, a directory only {{ ooniapi_gateway_deploy_user }} +# (and root) can read -- see tasks/main.yml for why that, and not shared +# /tmp, is where the per-secret files (each a Compose file-based secret, +# mounted read-only into its container -- never an environment variable) +# land. +# # Caveat: sudoers glob matching (fnmatch without FNM_PATHNAME) lets "*" -# match "/". The mv sources below are always files this same job just -# scp'd into /tmp itself, so this is a theoretical rather than practical -# widening of scope, but worth knowing if you're auditing this file. +# match "/", so this is a theoretical rather than practical widening of +# scope, but worth knowing if you're auditing this file. -Cmnd_Alias OONIAPI_DEPLOY_MV_UNIT = /usr/bin/mv /tmp/*.container /etc/containers/systemd/*.container -Cmnd_Alias OONIAPI_DEPLOY_MV_UPSTREAM = /usr/bin/mv /tmp/*-upstream.conf /etc/nginx/conf.d/*-upstream.conf -Cmnd_Alias OONIAPI_DEPLOY_SYSTEMCTL = /usr/bin/systemctl daemon-reload, /usr/bin/systemctl restart *-a.service, /usr/bin/systemctl restart *-b.service, /usr/bin/systemctl reload nginx +Cmnd_Alias OONIAPI_DEPLOY_MV_COMPOSE = /usr/bin/mv {{ ooniapi_gateway_staging_dir }}/*.yaml /etc/ooniapi/*/*.yaml +Cmnd_Alias OONIAPI_DEPLOY_MV_UPSTREAM = /usr/bin/mv {{ ooniapi_gateway_staging_dir }}/*-upstream.conf /etc/nginx/conf.d/*-upstream.conf +Cmnd_Alias OONIAPI_DEPLOY_SECRET_INSTALL = /usr/bin/install -D -m 600 -o root -g root {{ ooniapi_gateway_staging_dir }}/*.secret /etc/ooniapi/*/*/secrets/* +Cmnd_Alias OONIAPI_DEPLOY_COMPOSE_UP = /usr/bin/docker compose -f /etc/ooniapi/*/*.yaml up -d --pull always --remove-orphans +Cmnd_Alias OONIAPI_DEPLOY_NGINX_RELOAD = /usr/bin/systemctl reload nginx Cmnd_Alias OONIAPI_DEPLOY_NGINX_TEST = /usr/sbin/nginx -t Cmnd_Alias OONIAPI_DEPLOY_MARKER = /usr/bin/tee /etc/ooniapi/*/active_slot -Cmnd_Alias OONIAPI_DEPLOY_PODMAN_SECRET = /usr/bin/podman secret create --replace * - -{{ ooniapi_gateway_deploy_user }} ALL=(root) NOPASSWD: OONIAPI_DEPLOY_MV_UNIT, OONIAPI_DEPLOY_MV_UPSTREAM, OONIAPI_DEPLOY_SYSTEMCTL, OONIAPI_DEPLOY_NGINX_TEST, OONIAPI_DEPLOY_MARKER -{{ ooniapi_gateway_deploy_user }} ALL=(root) NOPASSWD: OONIAPI_DEPLOY_PODMAN_SECRET +{{ ooniapi_gateway_deploy_user }} ALL=(root) NOPASSWD: OONIAPI_DEPLOY_MV_COMPOSE, OONIAPI_DEPLOY_MV_UPSTREAM, OONIAPI_DEPLOY_SECRET_INSTALL, OONIAPI_DEPLOY_COMPOSE_UP, OONIAPI_DEPLOY_NGINX_RELOAD, OONIAPI_DEPLOY_NGINX_TEST, OONIAPI_DEPLOY_MARKER diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 943e71a4..31805934 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -253,14 +253,15 @@ data "aws_ssm_parameter" "account_id_hashing_key" { name = "/oonidevops/secrets/ooni_services/account_id_hashing_key" } -### Blue/green deploy secrets (Podman Quadlet on dedicated Hetzner hosts) +### Blue/green deploy secrets (Docker + systemd on dedicated Hetzner hosts) # # These mirror the values already passed as `task_secrets` to the ECS task # definitions above, but consolidated into one JSON blob per service so the # "blue_green" deploy_mode CodeBuild job can fetch them in a single -# secretsmanager:GetSecretValue call and push each key as a Podman secret. -# Values are pulled from the same underlying data sources used by the ECS -# `task_secrets` maps, so both deploy paths stay in sync automatically. +# secretsmanager:GetSecretValue call and write each key into the target +# slot's env file. Values are pulled from the same underlying data sources +# used by the ECS `task_secrets` maps, so both deploy paths stay in sync +# automatically. data "aws_secretsmanager_secret_version" "ooniapi_user_access_key_id" { secret_id = module.ooniapi_user.aws_access_key_id_arn @@ -647,7 +648,7 @@ EOF module "ooniapi_ooniprobe_deployer" { source = "../../modules/ooniapi_service_deployer" - # Flip to "blue_green" to switch this service to the Podman Quadlet + # Flip to "blue_green" to switch this service to the Docker + systemd # blue/green deploy on the dedicated Hetzner hosts. deploy_mode = "ecs" @@ -676,9 +677,9 @@ module "ooniapi_ooniprobe_deployer" { ANONC_MANIFEST_BUCKET = aws_s3_bucket.anoncred_manifests.bucket ANONC_MANIFEST_FILE = "manifest.json" } - secrets = keys(local.ooniapi_deploy_service_secrets.ooniprobe) - service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["ooniprobe"].arn - deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn + secrets = keys(local.ooniapi_deploy_service_secrets.ooniprobe) + service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["ooniprobe"].arn + deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn } module "ooniapi_ooniprobe" { @@ -744,7 +745,7 @@ module "ooniapi_ooniprobe" { module "ooniapi_reverseproxy_deployer" { source = "../../modules/ooniapi_service_deployer" - # Flip to "blue_green" to switch this service to the Podman Quadlet + # Flip to "blue_green" to switch this service to the Docker + systemd # blue/green deploy on the dedicated Hetzner hosts. deploy_mode = "ecs" @@ -765,9 +766,9 @@ module "ooniapi_reverseproxy_deployer" { env_vars = { TARGET_URL = "https://backend-hel.ooni.org/" } - secrets = keys(local.ooniapi_deploy_service_secrets.reverseproxy) - service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["reverseproxy"].arn - deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn + secrets = keys(local.ooniapi_deploy_service_secrets.reverseproxy) + service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["reverseproxy"].arn + deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn } module "ooniapi_reverseproxy" { @@ -1063,7 +1064,7 @@ module "fastpath_builder" { module "ooniapi_oonirun_deployer" { source = "../../modules/ooniapi_service_deployer" - # Flip to "blue_green" to switch this service to the Podman Quadlet + # Flip to "blue_green" to switch this service to the Docker + systemd # blue/green deploy on the dedicated Hetzner hosts. deploy_mode = "ecs" @@ -1081,10 +1082,10 @@ module "ooniapi_oonirun_deployer" { ecs_cluster_name = module.ooniapi_cluster.cluster_name # Pre-wired for the future flip to deploy_mode = "blue_green" - env_vars = {} - secrets = keys(local.ooniapi_deploy_service_secrets.oonirun) - service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["oonirun"].arn - deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn + env_vars = {} + secrets = keys(local.ooniapi_deploy_service_secrets.oonirun) + service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["oonirun"].arn + deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn } module "ooniapi_oonirun" { @@ -1124,7 +1125,7 @@ module "ooniapi_oonirun" { module "ooniapi_oonifindings_deployer" { source = "../../modules/ooniapi_service_deployer" - # Flip to "blue_green" to switch this service to the Podman Quadlet + # Flip to "blue_green" to switch this service to the Docker + systemd # blue/green deploy on the dedicated Hetzner hosts. deploy_mode = "ecs" @@ -1142,10 +1143,10 @@ module "ooniapi_oonifindings_deployer" { ecs_cluster_name = module.ooniapi_cluster.cluster_name # Pre-wired for the future flip to deploy_mode = "blue_green" - env_vars = {} - secrets = keys(local.ooniapi_deploy_service_secrets.oonifindings) - service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["oonifindings"].arn - deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn + env_vars = {} + secrets = keys(local.ooniapi_deploy_service_secrets.oonifindings) + service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["oonifindings"].arn + deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn } module "ooniapi_oonifindings" { @@ -1185,7 +1186,7 @@ module "ooniapi_oonifindings" { module "ooniapi_ooniauth_deployer" { source = "../../modules/ooniapi_service_deployer" - # Flip to "blue_green" to switch this service to the Podman Quadlet + # Flip to "blue_green" to switch this service to the Docker + systemd # blue/green deploy on the dedicated Hetzner hosts. deploy_mode = "ecs" @@ -1219,9 +1220,9 @@ module "ooniapi_ooniauth_deployer" { "contact@openobservatory.org" ]) } - secrets = keys(local.ooniapi_deploy_service_secrets.ooniauth) - service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["ooniauth"].arn - deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn + secrets = keys(local.ooniapi_deploy_service_secrets.ooniauth) + service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["ooniauth"].arn + deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn } module "ooniapi_ooniauth" { @@ -1279,7 +1280,7 @@ module "ooniapi_ooniauth" { module "ooniapi_oonimeasurements_deployer" { source = "../../modules/ooniapi_service_deployer" - # Flip to "blue_green" to switch this service to the Podman Quadlet + # Flip to "blue_green" to switch this service to the Docker + systemd # blue/green deploy on the dedicated Hetzner hosts. deploy_mode = "ecs" @@ -1304,9 +1305,9 @@ module "ooniapi_oonimeasurements_deployer" { RATE_LIMITS_WHITELISTED_IPADDRS = jsonencode(["5.9.112.244"]) RATE_LIMITS_UNMETERED_PAGES = jsonencode(["/metrics", "/health"]) } - secrets = keys(local.ooniapi_deploy_service_secrets.oonimeasurements) - service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["oonimeasurements"].arn - deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn + secrets = keys(local.ooniapi_deploy_service_secrets.oonimeasurements) + service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["oonimeasurements"].arn + deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn ecs_service_name = module.ooniapi_oonimeasurements.ecs_service_name ecs_cluster_name = module.oonitier1plus_cluster.cluster_name diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf index dfa6de02..f3543a25 100644 --- a/tf/environments/prod/main.tf +++ b/tf/environments/prod/main.tf @@ -279,14 +279,15 @@ resource "aws_secretsmanager_secret_version" "oonipg_url" { ) } -### Blue/green deploy secrets (Podman Quadlet on dedicated Hetzner hosts) +### Blue/green deploy secrets (Docker + systemd on dedicated Hetzner hosts) # # These mirror the values already passed as `task_secrets` to the ECS task # definitions above, but consolidated into one JSON blob per service so the # "blue_green" deploy_mode CodeBuild job can fetch them in a single -# secretsmanager:GetSecretValue call and push each key as a Podman secret. -# Values are pulled from the same underlying data sources used by the ECS -# `task_secrets` maps, so both deploy paths stay in sync automatically. +# secretsmanager:GetSecretValue call and write each key into the target +# slot's env file. Values are pulled from the same underlying data sources +# used by the ECS `task_secrets` maps, so both deploy paths stay in sync +# automatically. data "aws_secretsmanager_secret_version" "ooniapi_user_access_key_id" { secret_id = module.ooniapi_user.aws_access_key_id_arn @@ -522,7 +523,7 @@ module "ooni_th_droplet" { module "ooniapi_reverseproxy_deployer" { source = "../../modules/ooniapi_service_deployer" - # Flip to "blue_green" to switch this service to the Podman Quadlet + # Flip to "blue_green" to switch this service to the Docker + systemd # blue/green deploy on the dedicated Hetzner hosts. deploy_mode = "ecs" @@ -958,7 +959,7 @@ EOF module "ooniapi_ooniprobe_deployer" { source = "../../modules/ooniapi_service_deployer" - # Flip to "blue_green" to switch this service to the Podman Quadlet + # Flip to "blue_green" to switch this service to the Docker + systemd # blue/green deploy on the dedicated Hetzner hosts. deploy_mode = "ecs" @@ -1149,7 +1150,7 @@ module "fastpath_builder" { module "ooniapi_oonirun_deployer" { source = "../../modules/ooniapi_service_deployer" - # Flip to "blue_green" to switch this service to the Podman Quadlet + # Flip to "blue_green" to switch this service to the Docker + systemd # blue/green deploy on the dedicated Hetzner hosts. deploy_mode = "ecs" @@ -1167,10 +1168,10 @@ module "ooniapi_oonirun_deployer" { ecs_cluster_name = module.ooniapi_cluster.cluster_name # Pre-wired for the future flip to deploy_mode = "blue_green" - env_vars = {} - secrets = keys(local.ooniapi_deploy_service_secrets.oonirun) - service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["oonirun"].arn - deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn + env_vars = {} + secrets = keys(local.ooniapi_deploy_service_secrets.oonirun) + service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["oonirun"].arn + deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn } module "ooniapi_oonirun" { @@ -1211,7 +1212,7 @@ module "ooniapi_oonirun" { module "ooniapi_oonifindings_deployer" { source = "../../modules/ooniapi_service_deployer" - # Flip to "blue_green" to switch this service to the Podman Quadlet + # Flip to "blue_green" to switch this service to the Docker + systemd # blue/green deploy on the dedicated Hetzner hosts. deploy_mode = "ecs" @@ -1230,9 +1231,9 @@ module "ooniapi_oonifindings_deployer" { # Pre-wired for the future flip to deploy_mode = "blue_green" env_vars = {} - secrets = keys(local.ooniapi_deploy_service_secrets.oonifindings) - service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["oonifindings"].arn - deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn + secrets = keys(local.ooniapi_deploy_service_secrets.oonifindings) + service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["oonifindings"].arn + deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn } module "ooniapi_oonifindings" { @@ -1273,7 +1274,7 @@ module "ooniapi_oonifindings" { module "ooniapi_ooniauth_deployer" { source = "../../modules/ooniapi_service_deployer" - # Flip to "blue_green" to switch this service to the Podman Quadlet + # Flip to "blue_green" to switch this service to the Docker + systemd # blue/green deploy on the dedicated Hetzner hosts. deploy_mode = "ecs" @@ -1307,9 +1308,9 @@ module "ooniapi_ooniauth_deployer" { "contact@openobservatory.org" ]) } - secrets = keys(local.ooniapi_deploy_service_secrets.ooniauth) - service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["ooniauth"].arn - deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn + secrets = keys(local.ooniapi_deploy_service_secrets.ooniauth) + service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["ooniauth"].arn + deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn } module "ooniapi_ooniauth" { @@ -1369,7 +1370,7 @@ module "ooniapi_ooniauth" { module "ooniapi_oonimeasurements_deployer" { source = "../../modules/ooniapi_service_deployer" - # Flip to "blue_green" to switch this service to the Podman Quadlet + # Flip to "blue_green" to switch this service to the Docker + systemd # blue/green deploy on the dedicated Hetzner hosts. deploy_mode = "ecs" @@ -1398,9 +1399,9 @@ module "ooniapi_oonimeasurements_deployer" { RATE_LIMITS_WHITELISTED_IPADDRS = jsonencode(["5.9.112.244"]) RATE_LIMITS_UNMETERED_PAGES = jsonencode(["/metrics", "/health"]) } - secrets = keys(local.ooniapi_deploy_service_secrets.oonimeasurements) - service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["oonimeasurements"].arn - deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn + secrets = keys(local.ooniapi_deploy_service_secrets.oonimeasurements) + service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["oonimeasurements"].arn + deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn } module "ooniapi_oonimeasurements" { diff --git a/tf/modules/ooniapi_service_deployer/files/deploy.py b/tf/modules/ooniapi_service_deployer/files/deploy.py new file mode 100644 index 00000000..57cbf5c7 --- /dev/null +++ b/tf/modules/ooniapi_service_deployer/files/deploy.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +# Blue/green deploy for a single OONI API service, run by the +# ooniapi--deploy CodeBuild project (see ../main.tf and +# ./buildspec_deploy.yml). Stdlib only -- no pip installs, so the CodeBuild +# image needs nothing beyond python3, ssh/scp and the aws cli, all of which +# it already ships. +# +# For each of DEPLOY_HOST_PRIMARY/SECONDARY: find the currently idle slot +# (a/b) on that host, write that slot's secrets as individual files (picked +# up by the compose file's file-based `secrets:` entries -- the container +# only ever sees these mounted read-only at /run/secrets/, never as +# environment variables), install & (re)start that slot's docker-compose.yaml +# at the new image tag, health-check it directly on its host port, then flip +# nginx's upstream to it and record the new active slot. +# +# Containers are started with `restart: always`, so the docker daemon +# itself keeps them running across crashes/reboots -- no systemd unit is +# needed on top of docker compose. + +import json +import os +import subprocess +import sys +import tempfile +import time + +SSH_OPTS = [ + "-o", "StrictHostKeyChecking=accept-new", + "-o", "ConnectTimeout=10", + "-i", "/tmp/deploy_key", +] + +# Private, deploy-user-owned (0700) directory files land in before being +# moved/installed into place -- not shared /tmp, so the env file (which +# carries service secrets) is never briefly world-readable on the target +# host. Must match ooniapi_gateway_staging_dir in the Ansible role. +STAGING_DIR = "/var/lib/ooniapi/deploy-staging" + + +def require_env(name): + value = os.environ.get(name) + if not value: + sys.exit(f"missing required environment variable: {name}") + return value + + +def run(cmd, **kwargs): + return subprocess.run(cmd, check=True, text=True, **kwargs) + + +def ssh(user, host, remote_cmd): + return run(["ssh", *SSH_OPTS, f"{user}@{host}", remote_cmd]) + + +def ssh_output(user, host, remote_cmd): + return run(["ssh", *SSH_OPTS, f"{user}@{host}", remote_cmd], capture_output=True).stdout + + +def ssh_succeeds(user, host, remote_cmd): + return subprocess.run(["ssh", *SSH_OPTS, f"{user}@{host}", remote_cmd]).returncode == 0 + + +def scp(user, host, local_path, remote_path): + # -p preserves the local file's mode, so a locally-chmod'd-600 file + # (e.g. the env file) lands on the target host already locked down + # instead of briefly readable at whatever the remote umask would give it. + run(["scp", *SSH_OPTS, "-p", local_path, f"{user}@{host}:{remote_path}"]) + + +def s3_fetch(bucket, key): + path = os.path.join(tempfile.gettempdir(), os.path.basename(key)) + run(["aws", "s3", "cp", f"s3://{bucket}/{key}", path]) + with open(path) as f: + return f.read() + + +def secretsmanager_get(secret_arn): + return run( + ["aws", "secretsmanager", "get-secret-value", "--secret-id", secret_arn, + "--query", "SecretString", "--output", "text"], + capture_output=True, + ).stdout + + +def write_tmp(name, content, mode=0o644): + path = os.path.join(tempfile.gettempdir(), name) + with open(path, "w") as f: + f.write(content) + os.chmod(path, mode) + return path + + +def deploy_host(host, ctx): + service = ctx["service"] + user = ctx["user"] + print(f"=== {service}: deploying to {host} ===") + + active_slot = ssh_output( + user, host, f"cat /etc/ooniapi/{service}/active_slot 2>/dev/null || echo a" + ).strip() or "a" + target_slot = "b" if active_slot == "a" else "a" + target_port = ctx["host_port_b"] if target_slot == "b" else ctx["host_port_a"] + print(f"{service} on {host}: active slot is {active_slot}, deploying to slot {target_slot} (port {target_port})") + + # compose file for the target slot + compose_file = f"{service}-{target_slot}.yaml" + compose_path = f"/etc/ooniapi/{service}/{compose_file}" + compose_content = s3_fetch(ctx["bucket"], f"{service}/{compose_file}") + compose_content = compose_content.replace("__IMAGE_TAG__", ctx["image_tag"]) + compose_local = write_tmp(compose_file, compose_content) + scp(user, host, compose_local, f"{STAGING_DIR}/{compose_file}") + ssh(user, host, f"sudo mv {STAGING_DIR}/{compose_file} {compose_path}") + + # one file per secret, chmod'd 600 locally before scp -p sends it, then + # installed straight to the path the compose file's `secrets:` section + # references (install -D creates the slot's secrets/ dir on first use) + for key, value in ctx["secrets"].items(): + secret_file = f"{service}-{target_slot}-{key}.secret" + secret_local = write_tmp(secret_file, value, mode=0o600) + secret_dest = f"/etc/ooniapi/{service}/{target_slot}/secrets/{key}" + scp(user, host, secret_local, f"{STAGING_DIR}/{secret_file}") + ssh(user, host, + f"sudo install -D -m 600 -o root -g root {STAGING_DIR}/{secret_file} {secret_dest}" + f" && rm -f {STAGING_DIR}/{secret_file}") + + ssh(user, host, f"sudo docker compose -f {compose_path} up -d --pull always --remove-orphans") + + healthy = False + for _ in range(10): + if ssh_succeeds(user, host, f"curl -sf -o /dev/null http://127.0.0.1:{target_port}/health"): + healthy = True + break + time.sleep(2) + if not healthy: + sys.exit(f"{service} on {host}: slot {target_slot} failed health check, aborting deploy") + + state_a, state_b = ("", "down") if target_slot == "a" else ("down", "") + upstream_file = f"{service}-upstream.conf" + upstream_content = s3_fetch(ctx["bucket"], f"{service}/{upstream_file}") + upstream_content = upstream_content.replace("__STATE_A__", state_a).replace("__STATE_B__", state_b) + upstream_local = write_tmp(upstream_file, upstream_content) + scp(user, host, upstream_local, f"{STAGING_DIR}/{upstream_file}") + ssh(user, host, + f"sudo mv {STAGING_DIR}/{upstream_file} /etc/nginx/conf.d/{upstream_file}" + f" && sudo nginx -t && sudo systemctl reload nginx") + ssh(user, host, f"echo {target_slot} | sudo tee /etc/ooniapi/{service}/active_slot > /dev/null") + + print(f"=== {service} on {host}: now serving from slot {target_slot} ===") + + +def main(): + with open("imagedefinitions.json") as f: + image_tag = json.load(f)[0]["imageUri"].rsplit(":", 1)[-1] + + service = require_env("SERVICE_NAME") + print(f"Deploying {service} image tag {image_tag}") + + with open("/tmp/deploy_key", "w") as f: + f.write(secretsmanager_get(require_env("DEPLOY_SSH_KEY_SECRET_ARN"))) + os.chmod("/tmp/deploy_key", 0o600) + + ctx = { + "service": service, + "user": require_env("DEPLOY_SSH_USER"), + "bucket": require_env("DEPLOY_BUCKET"), + "image_tag": image_tag, + "host_port_a": require_env("HOST_PORT_A"), + "host_port_b": require_env("HOST_PORT_B"), + "secrets": json.loads(secretsmanager_get(require_env("SERVICE_SECRETS_ARN"))), + } + + for host in (require_env("DEPLOY_HOST_PRIMARY"), require_env("DEPLOY_HOST_SECONDARY")): + deploy_host(host, ctx) + + +if __name__ == "__main__": + main() diff --git a/tf/modules/ooniapi_service_deployer/main.tf b/tf/modules/ooniapi_service_deployer/main.tf index b95722b7..46e90700 100755 --- a/tf/modules/ooniapi_service_deployer/main.tf +++ b/tf/modules/ooniapi_service_deployer/main.tf @@ -161,16 +161,16 @@ resource "aws_codebuild_project" "ooniapi" { } } -## Podman Quadlet blue/green deploy (deploy_mode = "blue_green") +## Docker Compose blue/green deploy (deploy_mode = "blue_green") -resource "aws_s3_object" "quadlet_unit" { +resource "aws_s3_object" "compose_file" { for_each = var.deploy_mode == "blue_green" ? { a = var.host_port_a, b = var.host_port_b } : {} - bucket = var.quadlet_units_bucket - key = "${var.service_name}/${var.service_name}-${each.key}.container" + bucket = var.deploy_bucket + key = "${var.service_name}/${var.service_name}-${each.key}.yaml" content_type = "text/plain" - content = templatefile("${path.module}/templates/quadlet.container.tftpl", { + content = templatefile("${path.module}/templates/compose.yaml.tftpl", { service_name = var.service_name slot = each.key host_port = each.value @@ -184,7 +184,7 @@ resource "aws_s3_object" "quadlet_unit" { resource "aws_s3_object" "nginx_upstream" { count = var.deploy_mode == "blue_green" ? 1 : 0 - bucket = var.quadlet_units_bucket + bucket = var.deploy_bucket key = "${var.service_name}/${var.service_name}-upstream.conf" content_type = "text/plain" @@ -195,6 +195,16 @@ resource "aws_s3_object" "nginx_upstream" { }) } +resource "aws_s3_object" "deploy_script" { + count = var.deploy_mode == "blue_green" ? 1 : 0 + + bucket = var.deploy_bucket + key = "${var.service_name}/deploy.py" + content_type = "text/x-python" + source = "${path.module}/files/deploy.py" + etag = filemd5("${path.module}/files/deploy.py") +} + resource "aws_iam_policy" "deploy" { count = var.deploy_mode == "blue_green" ? 1 : 0 @@ -235,7 +245,7 @@ resource "aws_iam_policy" "deploy" { { Effect = "Allow" Action = ["s3:GetObject"] - Resource = ["arn:aws:s3:::${var.quadlet_units_bucket}/${var.service_name}/*"] + Resource = ["arn:aws:s3:::${var.deploy_bucket}/${var.service_name}/*"] }, { Effect = "Allow" @@ -314,16 +324,8 @@ resource "aws_codebuild_project" "deploy" { value = tostring(var.host_port_b) } environment_variable { - name = "CONTAINER_PORT" - value = tostring(var.container_port) - } - environment_variable { - name = "NETWORK_NAME" - value = var.network_name - } - environment_variable { - name = "QUADLET_BUCKET" - value = var.quadlet_units_bucket + name = "DEPLOY_BUCKET" + value = var.deploy_bucket } environment_variable { name = "DEPLOY_HOST_PRIMARY" diff --git a/tf/modules/ooniapi_service_deployer/templates/buildspec_deploy.yml b/tf/modules/ooniapi_service_deployer/templates/buildspec_deploy.yml index 1ea3968e..7b160911 100644 --- a/tf/modules/ooniapi_service_deployer/templates/buildspec_deploy.yml +++ b/tf/modules/ooniapi_service_deployer/templates/buildspec_deploy.yml @@ -1,92 +1,10 @@ version: 0.2 -env: - variables: - SSH_OPTS: "-o StrictHostKeyChecking=accept-new -o ConnectTimeout=10 -i /tmp/deploy_key" - phases: - install: - commands: - - command -v jq >/dev/null || (apt-get update -y && apt-get install -y jq) - pre_build: commands: - - aws secretsmanager get-secret-value --secret-id "$DEPLOY_SSH_KEY_SECRET_ARN" --query SecretString --output text > /tmp/deploy_key - - chmod 600 /tmp/deploy_key - - aws secretsmanager get-secret-value --secret-id "$SERVICE_SECRETS_ARN" --query SecretString --output text > /tmp/service_secrets.json - - IMAGE_URI=$(jq -r '.[0].imageUri' imagedefinitions.json) - - export IMAGE_TAG="${IMAGE_URI##*:}" - - echo "Deploying $SERVICE_NAME image tag $IMAGE_TAG to $DEPLOY_HOST_PRIMARY then $DEPLOY_HOST_SECONDARY" + - aws s3 cp "s3://${DEPLOY_BUCKET}/${SERVICE_NAME}/deploy.py" /tmp/deploy.py build: commands: - - | - set -euo pipefail - - deploy_host() { - HOST="$1" - echo "=== $SERVICE_NAME: deploying to $HOST ===" - - ACTIVE_SLOT=$(ssh $SSH_OPTS "${DEPLOY_SSH_USER}@${HOST}" "cat /etc/ooniapi/${SERVICE_NAME}/active_slot 2>/dev/null || echo a") - if [ "$ACTIVE_SLOT" = "a" ]; then - TARGET_SLOT=b - TARGET_PORT="$HOST_PORT_B" - else - TARGET_SLOT=a - TARGET_PORT="$HOST_PORT_A" - fi - echo "$SERVICE_NAME on $HOST: active slot is $ACTIVE_SLOT, deploying to slot $TARGET_SLOT (port $TARGET_PORT)" - - # a. + b. render/fetch the target slot's quadlet unit and install it - UNIT_FILE="${SERVICE_NAME}-${TARGET_SLOT}.container" - aws s3 cp "s3://${QUADLET_BUCKET}/${SERVICE_NAME}/${UNIT_FILE}" "/tmp/${UNIT_FILE}" - sed -i "s|__IMAGE_TAG__|${IMAGE_TAG}|g" "/tmp/${UNIT_FILE}" - - scp $SSH_OPTS "/tmp/${UNIT_FILE}" "${DEPLOY_SSH_USER}@${HOST}:/tmp/${UNIT_FILE}" - ssh $SSH_OPTS "${DEPLOY_SSH_USER}@${HOST}" "sudo mv /tmp/${UNIT_FILE} /etc/containers/systemd/${UNIT_FILE}" - - # c. push service secrets as podman secrets (values travel over - # stdin, never interpolated into the remote command string) - for KEY in $(jq -r 'keys[]' /tmp/service_secrets.json); do - VALUE=$(jq -r --arg k "$KEY" '.[$k]' /tmp/service_secrets.json) - printf '%s' "$VALUE" | ssh $SSH_OPTS "${DEPLOY_SSH_USER}@${HOST}" "sudo podman secret create --replace ${KEY} -" - done - - # d. reload systemd and (re)start only the target slot - ssh $SSH_OPTS "${DEPLOY_SSH_USER}@${HOST}" "sudo systemctl daemon-reload && sudo systemctl restart ${SERVICE_NAME}-${TARGET_SLOT}.service" - - # e. health-check the target slot directly, bypassing nginx - HEALTHY="" - for i in $(seq 1 10); do - if ssh $SSH_OPTS "${DEPLOY_SSH_USER}@${HOST}" "curl -sf -o /dev/null http://127.0.0.1:${TARGET_PORT}/health"; then - HEALTHY=1 - break - fi - sleep 2 - done - if [ -z "$HEALTHY" ]; then - echo "$SERVICE_NAME on $HOST: slot $TARGET_SLOT failed health check, aborting deploy" - exit 1 - fi - - # f. flip nginx to the target slot and record the new active slot - if [ "$TARGET_SLOT" = "a" ]; then - STATE_A="" - STATE_B="down" - else - STATE_A="down" - STATE_B="" - fi - UPSTREAM_FILE="${SERVICE_NAME}-upstream.conf" - aws s3 cp "s3://${QUADLET_BUCKET}/${SERVICE_NAME}/${UPSTREAM_FILE}" "/tmp/${UPSTREAM_FILE}" - sed -i "s|__STATE_A__|${STATE_A}|g; s|__STATE_B__|${STATE_B}|g" "/tmp/${UPSTREAM_FILE}" - - scp $SSH_OPTS "/tmp/${UPSTREAM_FILE}" "${DEPLOY_SSH_USER}@${HOST}:/tmp/${UPSTREAM_FILE}" - ssh $SSH_OPTS "${DEPLOY_SSH_USER}@${HOST}" "sudo mv /tmp/${UPSTREAM_FILE} /etc/nginx/conf.d/${UPSTREAM_FILE} && sudo nginx -t && sudo systemctl reload nginx" - ssh $SSH_OPTS "${DEPLOY_SSH_USER}@${HOST}" "echo ${TARGET_SLOT} | sudo tee /etc/ooniapi/${SERVICE_NAME}/active_slot > /dev/null" - - echo "=== $SERVICE_NAME on $HOST: now serving from slot $TARGET_SLOT ===" - } - - deploy_host "$DEPLOY_HOST_PRIMARY" - deploy_host "$DEPLOY_HOST_SECONDARY" + - python3 /tmp/deploy.py diff --git a/tf/modules/ooniapi_service_deployer/templates/compose.yaml.tftpl b/tf/modules/ooniapi_service_deployer/templates/compose.yaml.tftpl new file mode 100644 index 00000000..e0223c16 --- /dev/null +++ b/tf/modules/ooniapi_service_deployer/templates/compose.yaml.tftpl @@ -0,0 +1,34 @@ +name: ${service_name}-${slot} + +networks: + default: + name: ${network_name} + external: true + +%{ if length(secrets) > 0 ~} +secrets: +%{ for name in secrets ~} + ${name}: + file: /etc/ooniapi/${service_name}/${slot}/secrets/${name} +%{ endfor ~} + +%{ endif ~} +services: + ${service_name}-${slot}: + image: docker.io/ooni/api-${service_name}:__IMAGE_TAG__ + container_name: ${service_name}-${slot} + restart: always + ports: + - "127.0.0.1:${host_port}:${container_port}" +%{ if length(env_vars) > 0 ~} + environment: +%{ for key, value in env_vars ~} + ${key}: ${jsonencode(value)} +%{ endfor ~} +%{ endif ~} +%{ if length(secrets) > 0 ~} + secrets: +%{ for name in secrets ~} + - ${name} +%{ endfor ~} +%{ endif ~} diff --git a/tf/modules/ooniapi_service_deployer/templates/quadlet.container.tftpl b/tf/modules/ooniapi_service_deployer/templates/quadlet.container.tftpl deleted file mode 100644 index f309d8c4..00000000 --- a/tf/modules/ooniapi_service_deployer/templates/quadlet.container.tftpl +++ /dev/null @@ -1,23 +0,0 @@ -[Unit] -Description=${service_name} (slot ${slot}) -After=network-online.target -Wants=network-online.target - -[Container] -ContainerName=${service_name}-${slot} -Image=docker.io/ooni/api-${service_name}:__IMAGE_TAG__ -Network=${network_name} -PublishPort=127.0.0.1:${host_port}:${container_port} -%{ for key, value in env_vars ~} -Environment="${key}=${value}" -%{ endfor ~} -%{ for secret_name in secrets ~} -Secret=${secret_name} -%{ endfor ~} - -[Service] -Restart=always -TimeoutStartSec=60 - -[Install] -WantedBy=multi-user.target diff --git a/tf/modules/ooniapi_service_deployer/variables.tf b/tf/modules/ooniapi_service_deployer/variables.tf index 67be2b53..b6056c78 100644 --- a/tf/modules/ooniapi_service_deployer/variables.tf +++ b/tf/modules/ooniapi_service_deployer/variables.tf @@ -39,7 +39,7 @@ variable "deploy_mode" { description = <<-EOF Which Deploy stage implementation the pipeline uses: - "ecs" (default) the existing ECS rolling-deploy stage. - - "blue_green" Podman Quadlet blue/green deploy to dedicated Hetzner + - "blue_green" Docker Compose blue/green deploy to dedicated Hetzner hosts, driven by a CodeBuild "Deploy" action over SSH. This is opt-in per service so unmigrated services keep working unchanged. EOF @@ -68,8 +68,8 @@ variable "ecs_service_name" { # --- deploy_mode = "blue_green" ----------------------------------------- -variable "quadlet_units_bucket" { - description = "S3 bucket that rendered Quadlet unit files and the nginx upstream conf snippet are uploaded to. Required when deploy_mode = \"blue_green\"." +variable "deploy_bucket" { + description = "S3 bucket that rendered compose files, the nginx upstream conf snippet, and deploy.py are uploaded to. Required when deploy_mode = \"blue_green\"." type = string default = null } @@ -93,25 +93,25 @@ variable "container_port" { } variable "network_name" { - description = "Podman network the service's containers attach to. Required when deploy_mode = \"blue_green\"." + description = "Docker network the service's containers attach to. Required when deploy_mode = \"blue_green\"." type = string default = null } -variable "secrets" { - description = "Names of Podman secrets (created/updated on the host during deploy) to mount into the container. Values live in var.service_secrets_arn." - type = list(string) - default = [] -} - variable "env_vars" { - description = "Cleartext environment variables for the container. Same shape as ooniapi_service's task_environment (map(string))." + description = "Cleartext environment variables for the container. Same shape as ooniapi_service's task_environment (map(string)). Baked directly into the rendered compose file, since none of this is sensitive." type = map(string) default = {} } +variable "secrets" { + description = "Names of the keys in var.service_secrets_arn's JSON blob. Declared in the rendered compose file as Compose file-based secrets, so each is mounted read-only at /run/secrets/ in the container instead of being exposed as an environment variable. Required when deploy_mode = \"blue_green\"." + type = list(string) + default = [] +} + variable "service_secrets_arn" { - description = "ARN of the Secrets Manager secret holding the service's runtime secrets as a flat JSON key/value object. Each key is pushed to the target hosts as a Podman secret during deploy. Required when deploy_mode = \"blue_green\"." + description = "ARN of the Secrets Manager secret holding the service's runtime secrets as a flat JSON key/value object. Every key becomes a Compose file-based secret written to disk during deploy (see var.secrets). Required when deploy_mode = \"blue_green\"." type = string default = null } From 124a863134e128f28ac1922b784e5ef61ae4f926 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Sat, 3 Oct 2026 00:59:35 +0000 Subject: [PATCH 05/24] Use per-service ClickHouse URLs in blue/green service secrets main replaced the shared clickhouse_write_url, clickhouse_readonly_url and clickhouse_readonly_test_url parameters with one per service (b7baff6). The blue/green service secrets are built from the same parameters as each service's ECS task_secrets, so they now point at the per-service ones again: ooniprobe, oonirun and oonimeasurements in dev, oonirun in prod. dev oonifindings no longer gets a CLICKHOUSE_URL, like its ECS task. Without this, terraform init fails on references to undeclared data sources after merging main. --- tf/environments/dev/main.tf | 7 +++---- tf/environments/prod/main.tf | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index c83c6ad1..8ded4312 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -284,20 +284,19 @@ locals { POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret_legacy.value PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value - CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_write_url.value + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_ooniprobe_url.value ANONC_SECRET_KEY = data.aws_ssm_parameter.anonc_secret_key.value } oonirun = { POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value - CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_readonly_url.value + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_oonirun_url.value } oonifindings = { POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value - CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_readonly_url.value } ooniauth = { POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value @@ -311,7 +310,7 @@ locals { POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value - CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_readonly_test_url.value + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_oonimeasurements_test_url.value ACCOUNT_ID_HASHING_KEY = data.aws_ssm_parameter.account_id_hashing_key.value } } diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf index 39589634..e0c4d2e1 100644 --- a/tf/environments/prod/main.tf +++ b/tf/environments/prod/main.tf @@ -321,7 +321,7 @@ locals { POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value - CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_readonly_url.value + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_oonirun_url.value } oonifindings = { POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value From 1a7bdd6249785f1192ac9a21de05b0824eb154fe Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Sat, 3 Oct 2026 01:01:55 +0000 Subject: [PATCH 06/24] deploy.py: log every command it runs Review: "We should add a log.info on this" (on run()). Every ssh, scp and aws command now goes through the logger with a timestamp, so the CodeBuild log shows what ran against which host and when, not just the start and end of each host's deploy. Secret values are only ever written to files, never put on a command line, so they don't reach the log. --- .../ooniapi_service_deployer/files/deploy.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/tf/modules/ooniapi_service_deployer/files/deploy.py b/tf/modules/ooniapi_service_deployer/files/deploy.py index 57cbf5c7..68ff53e3 100644 --- a/tf/modules/ooniapi_service_deployer/files/deploy.py +++ b/tf/modules/ooniapi_service_deployer/files/deploy.py @@ -18,6 +18,7 @@ # needed on top of docker compose. import json +import logging import os import subprocess import sys @@ -36,6 +37,8 @@ # host. Must match ooniapi_gateway_staging_dir in the Ansible role. STAGING_DIR = "/var/lib/ooniapi/deploy-staging" +log = logging.getLogger("deploy") + def require_env(name): value = os.environ.get(name) @@ -45,6 +48,9 @@ def require_env(name): def run(cmd, **kwargs): + # every remote and aws command goes through here; secret values are only + # ever written to files, never passed on a command line, so this is safe + log.info("running: %s", " ".join(cmd)) return subprocess.run(cmd, check=True, text=True, **kwargs) @@ -57,7 +63,9 @@ def ssh_output(user, host, remote_cmd): def ssh_succeeds(user, host, remote_cmd): - return subprocess.run(["ssh", *SSH_OPTS, f"{user}@{host}", remote_cmd]).returncode == 0 + cmd = ["ssh", *SSH_OPTS, f"{user}@{host}", remote_cmd] + log.info("running: %s", " ".join(cmd)) + return subprocess.run(cmd).returncode == 0 def scp(user, host, local_path, remote_path): @@ -93,14 +101,14 @@ def write_tmp(name, content, mode=0o644): def deploy_host(host, ctx): service = ctx["service"] user = ctx["user"] - print(f"=== {service}: deploying to {host} ===") + log.info(f"=== {service}: deploying to {host} ===") active_slot = ssh_output( user, host, f"cat /etc/ooniapi/{service}/active_slot 2>/dev/null || echo a" ).strip() or "a" target_slot = "b" if active_slot == "a" else "a" target_port = ctx["host_port_b"] if target_slot == "b" else ctx["host_port_a"] - print(f"{service} on {host}: active slot is {active_slot}, deploying to slot {target_slot} (port {target_port})") + log.info(f"{service} on {host}: active slot is {active_slot}, deploying to slot {target_slot} (port {target_port})") # compose file for the target slot compose_file = f"{service}-{target_slot}.yaml" @@ -145,15 +153,16 @@ def deploy_host(host, ctx): f" && sudo nginx -t && sudo systemctl reload nginx") ssh(user, host, f"echo {target_slot} | sudo tee /etc/ooniapi/{service}/active_slot > /dev/null") - print(f"=== {service} on {host}: now serving from slot {target_slot} ===") + log.info(f"=== {service} on {host}: now serving from slot {target_slot} ===") def main(): + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s", stream=sys.stdout) with open("imagedefinitions.json") as f: image_tag = json.load(f)[0]["imageUri"].rsplit(":", 1)[-1] service = require_env("SERVICE_NAME") - print(f"Deploying {service} image tag {image_tag}") + log.info(f"Deploying {service} image tag {image_tag}") with open("/tmp/deploy_key", "w") as f: f.write(secretsmanager_get(require_env("DEPLOY_SSH_KEY_SECRET_ARN"))) From ab096419aab1fc07bd32f0a23e30be26f70d429c Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Sat, 3 Oct 2026 01:02:44 +0000 Subject: [PATCH 07/24] deploy.py: give a new slot as long as ECS gives a new task Review: "We should make sure 20s is enough". It wasn't comparable to what the same images get on ECS: the target groups (ooniapi_service) check /health every 30 s with a 5 s timeout and want 2 passes in a row, so a new task has about a minute to come up. The blue/green check gave 20 s, a single pass sufficed, and curl had no timeout of its own. Now each check has curl --max-time 5, the slot must pass twice in a row, and it has health_check_timeout seconds (default 120, per service) to do so. The time it took is logged, so the deadline can be tuned per service from real deploys; at 2 hosts per deploy it stays well within the deploy job's 20 minute build timeout. --- .../ooniapi_service_deployer/files/deploy.py | 26 +++++++++++++------ tf/modules/ooniapi_service_deployer/main.tf | 4 +++ .../ooniapi_service_deployer/variables.tf | 6 +++++ 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/tf/modules/ooniapi_service_deployer/files/deploy.py b/tf/modules/ooniapi_service_deployer/files/deploy.py index 68ff53e3..426573fd 100644 --- a/tf/modules/ooniapi_service_deployer/files/deploy.py +++ b/tf/modules/ooniapi_service_deployer/files/deploy.py @@ -133,14 +133,23 @@ def deploy_host(host, ctx): ssh(user, host, f"sudo docker compose -f {compose_path} up -d --pull always --remove-orphans") - healthy = False - for _ in range(10): - if ssh_succeeds(user, host, f"curl -sf -o /dev/null http://127.0.0.1:{target_port}/health"): - healthy = True - break - time.sleep(2) - if not healthy: - sys.exit(f"{service} on {host}: slot {target_slot} failed health check, aborting deploy") + # like the ECS target groups' health check (ooniapi_service: timeout 5, + # healthy_threshold 2): each check may take up to 5 s, and the slot must + # pass twice in a row. ECS gives a new task about a minute to come up; + # the deadline defaults to twice that, and is logged so it can be tuned + started = time.monotonic() + passes = 0 + while passes < 2: + if ssh_succeeds(user, host, f"curl -sf --max-time 5 -o /dev/null http://127.0.0.1:{target_port}/health"): + passes += 1 + else: + passes = 0 + elapsed = time.monotonic() - started + if passes < 2 and elapsed > ctx["health_check_timeout"]: + sys.exit(f"{service} on {host}: slot {target_slot} not healthy after {elapsed:.0f}s, aborting deploy") + if passes < 2: + time.sleep(2) + log.info(f"{service} on {host}: slot {target_slot} healthy after {time.monotonic() - started:.0f}s") state_a, state_b = ("", "down") if target_slot == "a" else ("down", "") upstream_file = f"{service}-upstream.conf" @@ -175,6 +184,7 @@ def main(): "image_tag": image_tag, "host_port_a": require_env("HOST_PORT_A"), "host_port_b": require_env("HOST_PORT_B"), + "health_check_timeout": int(os.environ.get("HEALTH_CHECK_TIMEOUT") or 120), "secrets": json.loads(secretsmanager_get(require_env("SERVICE_SECRETS_ARN"))), } diff --git a/tf/modules/ooniapi_service_deployer/main.tf b/tf/modules/ooniapi_service_deployer/main.tf index 46e90700..7a0ea9c3 100755 --- a/tf/modules/ooniapi_service_deployer/main.tf +++ b/tf/modules/ooniapi_service_deployer/main.tf @@ -347,6 +347,10 @@ resource "aws_codebuild_project" "deploy" { name = "DEPLOY_SSH_KEY_SECRET_ARN" value = var.deploy_ssh_key_secret_arn } + environment_variable { + name = "HEALTH_CHECK_TIMEOUT" + value = tostring(var.health_check_timeout) + } } logs_config { diff --git a/tf/modules/ooniapi_service_deployer/variables.tf b/tf/modules/ooniapi_service_deployer/variables.tf index b6056c78..9db59596 100644 --- a/tf/modules/ooniapi_service_deployer/variables.tf +++ b/tf/modules/ooniapi_service_deployer/variables.tf @@ -86,6 +86,12 @@ variable "host_port_b" { default = null } +variable "health_check_timeout" { + description = "Seconds a new blue/green slot has to pass its /health check twice in a row before the deploy is aborted. The ECS target groups give a new task about 60 s." + type = number + default = 120 +} + variable "container_port" { description = "Port the service listens on inside the container. Required when deploy_mode = \"blue_green\"." type = number From 0e205281421fa29851377c1429e168112b8371b7 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Sat, 3 Oct 2026 01:03:42 +0000 Subject: [PATCH 08/24] deploy.py: restore the live upstream when nginx -t fails Review: "what happens if nginx -t fails". The new upstream conf was moved into conf.d before nginx -t, and left there when it failed. nginx kept serving its old config, but the next reload on that host (another service's deploy, a certificate renewal) would fail, and a restart would take every service on the host down. Now a failing nginx -t puts back the upstream of the slot that is still serving and checks it again, then aborts the deploy without reloading nginx or moving the active_slot marker. If nginx -t fails even then, the config is broken independently of this deploy, and the error says so. The restore uses the same mv the deploy user's sudoers already allows. --- .../ooniapi_service_deployer/files/deploy.py | 31 ++++++++++++++----- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/tf/modules/ooniapi_service_deployer/files/deploy.py b/tf/modules/ooniapi_service_deployer/files/deploy.py index 426573fd..0d851ce0 100644 --- a/tf/modules/ooniapi_service_deployer/files/deploy.py +++ b/tf/modules/ooniapi_service_deployer/files/deploy.py @@ -151,15 +151,30 @@ def deploy_host(host, ctx): time.sleep(2) log.info(f"{service} on {host}: slot {target_slot} healthy after {time.monotonic() - started:.0f}s") - state_a, state_b = ("", "down") if target_slot == "a" else ("down", "") upstream_file = f"{service}-upstream.conf" - upstream_content = s3_fetch(ctx["bucket"], f"{service}/{upstream_file}") - upstream_content = upstream_content.replace("__STATE_A__", state_a).replace("__STATE_B__", state_b) - upstream_local = write_tmp(upstream_file, upstream_content) - scp(user, host, upstream_local, f"{STAGING_DIR}/{upstream_file}") - ssh(user, host, - f"sudo mv {STAGING_DIR}/{upstream_file} /etc/nginx/conf.d/{upstream_file}" - f" && sudo nginx -t && sudo systemctl reload nginx") + upstream_template = s3_fetch(ctx["bucket"], f"{service}/{upstream_file}") + + def install_upstream(live_slot): + state_a, state_b = ("", "down") if live_slot == "a" else ("down", "") + content = upstream_template.replace("__STATE_A__", state_a).replace("__STATE_B__", state_b) + scp(user, host, write_tmp(upstream_file, content), f"{STAGING_DIR}/{upstream_file}") + ssh(user, host, f"sudo mv {STAGING_DIR}/{upstream_file} /etc/nginx/conf.d/{upstream_file}") + + install_upstream(target_slot) + if not ssh_succeeds(user, host, "sudo nginx -t"): + # nginx is still running its old config, but a config that fails + # nginx -t must not stay in conf.d: the next reload (another + # service's deploy, a certificate renewal) would fail, and a restart + # would take every service on this host down. Put back the upstream + # of the slot still serving, which nginx -t accepted before. + log.error(f"{service} on {host}: nginx -t failed with slot {target_slot}'s upstream, restoring slot {active_slot}'s") + install_upstream(active_slot) + if not ssh_succeeds(user, host, "sudo nginx -t"): + sys.exit(f"{service} on {host}: nginx -t still fails after restoring slot {active_slot}'s upstream:" + " the nginx config is broken independently of this deploy, fix it before reloading nginx") + sys.exit(f"{service} on {host}: nginx -t failed with slot {target_slot}'s upstream; restored slot" + f" {active_slot}'s, which is still serving. Aborting deploy") + ssh(user, host, "sudo systemctl reload nginx") ssh(user, host, f"echo {target_slot} | sudo tee /etc/ooniapi/{service}/active_slot > /dev/null") log.info(f"=== {service} on {host}: now serving from slot {target_slot} ===") From 608d3f0a9b1b379849cd959401b00be38a0a9125 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Sat, 3 Oct 2026 01:04:58 +0000 Subject: [PATCH 09/24] Bound how long a slot taken out of rotation drains Review: "We should make sure there is some kind of timeout on connections while draining". After a flip, nginx workers from before the reload keep serving their in-flight requests to the old slot, with nothing bounding them but proxy_read_timeout (900 s), and long lived HTTP/2 connections can keep them longer. On ECS the ALB bounds the same thing with its deregistration delay, the AWS default of 300 s here. - ooniapi_gateway sets worker_shutdown_timeout to ooniapi_gateway_drain_timeout (300 s), through modules-enabled/, the only place included in nginx's main context. - The next deploy of a service replaces the slot that is draining, so deploy.py now waits until drain_timeout (300 s, per service) has passed since the previous flip, from the active_slot marker's mtime, before restarting it. A first deploy has no marker and does not wait. --- ansible/roles/ooniapi_gateway/defaults/main.yml | 7 +++++++ ansible/roles/ooniapi_gateway/tasks/main.yml | 14 ++++++++++++++ .../ooniapi_service_deployer/files/deploy.py | 14 ++++++++++++++ tf/modules/ooniapi_service_deployer/main.tf | 4 ++++ tf/modules/ooniapi_service_deployer/variables.tf | 6 ++++++ 5 files changed, 45 insertions(+) diff --git a/ansible/roles/ooniapi_gateway/defaults/main.yml b/ansible/roles/ooniapi_gateway/defaults/main.yml index 0b28f33b..4f910dc7 100644 --- a/ansible/roles/ooniapi_gateway/defaults/main.yml +++ b/ansible/roles/ooniapi_gateway/defaults/main.yml @@ -66,3 +66,10 @@ ooniapi_gateway_deploy_user: deploy # moved/installed into place. Must match STAGING_DIR in # tf/modules/ooniapi_service_deployer/files/deploy.py. ooniapi_gateway_staging_dir: /var/lib/ooniapi/deploy-staging + +# How long nginx workers left over from a reload may keep serving in-flight +# requests (to the slot a deploy just took out of rotation) before they are +# closed. Matches the ALB's deregistration delay on ECS (AWS default, 300 s), +# and must match drain_timeout in ooniapi_service_deployer: a deploy waits +# this long after the previous flip before it restarts the drained slot. +ooniapi_gateway_drain_timeout: 300 diff --git a/ansible/roles/ooniapi_gateway/tasks/main.yml b/ansible/roles/ooniapi_gateway/tasks/main.yml index 020790d3..2a9ac98a 100644 --- a/ansible/roles/ooniapi_gateway/tasks/main.yml +++ b/ansible/roles/ooniapi_gateway/tasks/main.yml @@ -203,6 +203,20 @@ mode: "0440" validate: "visudo -cf %s" +# worker_shutdown_timeout is only valid in nginx's main context, which +# conf.d can't reach; modules-enabled/*.conf is included there by both the +# nginx role's nginx.conf and Debian's stock one. +- name: Bound how long old nginx workers drain after a reload + ansible.builtin.copy: + dest: /etc/nginx/modules-enabled/90-ooniapi-gateway-drain.conf + content: | + # Managed by ansible - roles/ooniapi_gateway (ooniapi_gateway_drain_timeout) + worker_shutdown_timeout {{ ooniapi_gateway_drain_timeout }}s; + owner: root + group: root + mode: "0644" + notify: reload ooniapi gateway nginx + - name: Render the gateway vhost ansible.builtin.template: src: gateway.conf.j2 diff --git a/tf/modules/ooniapi_service_deployer/files/deploy.py b/tf/modules/ooniapi_service_deployer/files/deploy.py index 0d851ce0..13892537 100644 --- a/tf/modules/ooniapi_service_deployer/files/deploy.py +++ b/tf/modules/ooniapi_service_deployer/files/deploy.py @@ -106,6 +106,19 @@ def deploy_host(host, ctx): active_slot = ssh_output( user, host, f"cat /etc/ooniapi/{service}/active_slot 2>/dev/null || echo a" ).strip() or "a" + + # the slot about to be replaced is the one the previous deploy took out + # of rotation: nginx workers from before that flip may still be serving + # requests to it, for up to drain_timeout (worker_shutdown_timeout in + # the ooniapi_gateway role). Don't restart it under them. + since_flip = int(ssh_output( + user, host, + f"echo $(( $(date +%s) - $(stat -c %Y /etc/ooniapi/{service}/active_slot 2>/dev/null || echo 0) ))", + ).strip()) + wait = ctx["drain_timeout"] - since_flip + if wait > 0: + log.info(f"{service} on {host}: previous deploy flipped {since_flip}s ago, waiting {wait}s for its old slot to drain") + time.sleep(wait) target_slot = "b" if active_slot == "a" else "a" target_port = ctx["host_port_b"] if target_slot == "b" else ctx["host_port_a"] log.info(f"{service} on {host}: active slot is {active_slot}, deploying to slot {target_slot} (port {target_port})") @@ -200,6 +213,7 @@ def main(): "host_port_a": require_env("HOST_PORT_A"), "host_port_b": require_env("HOST_PORT_B"), "health_check_timeout": int(os.environ.get("HEALTH_CHECK_TIMEOUT") or 120), + "drain_timeout": int(os.environ.get("DRAIN_TIMEOUT") or 300), "secrets": json.loads(secretsmanager_get(require_env("SERVICE_SECRETS_ARN"))), } diff --git a/tf/modules/ooniapi_service_deployer/main.tf b/tf/modules/ooniapi_service_deployer/main.tf index 7a0ea9c3..5fbc02b3 100755 --- a/tf/modules/ooniapi_service_deployer/main.tf +++ b/tf/modules/ooniapi_service_deployer/main.tf @@ -351,6 +351,10 @@ resource "aws_codebuild_project" "deploy" { name = "HEALTH_CHECK_TIMEOUT" value = tostring(var.health_check_timeout) } + environment_variable { + name = "DRAIN_TIMEOUT" + value = tostring(var.drain_timeout) + } } logs_config { diff --git a/tf/modules/ooniapi_service_deployer/variables.tf b/tf/modules/ooniapi_service_deployer/variables.tf index 9db59596..5ce31a1e 100644 --- a/tf/modules/ooniapi_service_deployer/variables.tf +++ b/tf/modules/ooniapi_service_deployer/variables.tf @@ -92,6 +92,12 @@ variable "health_check_timeout" { default = 120 } +variable "drain_timeout" { + description = "Seconds requests may keep draining from a blue/green slot after it is taken out of rotation; must match ooniapi_gateway_drain_timeout on the hosts. A deploy waits this long after the previous one before restarting that slot. Mirrors the ALB deregistration delay (AWS default, 300 s)." + type = number + default = 300 +} + variable "container_port" { description = "Port the service listens on inside the container. Required when deploy_mode = \"blue_green\"." type = number From 16aa56bbbde02cbc96f8fd37110fb9a192195c40 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Sat, 3 Oct 2026 01:18:27 +0000 Subject: [PATCH 10/24] Declare the ooniapi ALB routes in routes.yaml The routes of the API are going to be served by two frontends, the ALB for the services on ECS and the nginx gateway on the Hetzner hosts for those deployed blue/green, and their copies have already drifted: the gateway's hand written locations predate main's X-Protocol-Version rules. routes.yaml becomes the one place they are declared; this change has Terraform read it, the gateway follows. The 18 hand written listener rules become one aws_lb_listener_rule with for_each over routes.yaml, with moved blocks from their old addresses. Planned with fake ARNs (no state), the old and the new code produce the same 18 rules: same priorities, targets, conditions and tags. Applying should only move them in the state. The "hotfix" that made the oonimeasurements and testlists rules optional (count = arn != null ? 1 : 0) is removed rather than carried over. It had stopped working anyway: oonimeasurements_rule_3 lacked the count, so the frontend couldn't be planned without oonimeasurements. Both environments pass every target group, so those two inputs are now required and nullable = false. A missing service fails the plan with "Required variable not set" instead of silently losing its routes. The planned rules are unchanged. --- tf/modules/ooniapi_frontend/main.tf | 419 ++++++----------------- tf/modules/ooniapi_frontend/routes.yaml | 156 +++++++++ tf/modules/ooniapi_frontend/variables.tf | 4 +- 3 files changed, 254 insertions(+), 325 deletions(-) create mode 100644 tf/modules/ooniapi_frontend/routes.yaml diff --git a/tf/modules/ooniapi_frontend/main.tf b/tf/modules/ooniapi_frontend/main.tf index 2e1bee8e..8b6d0650 100644 --- a/tf/modules/ooniapi_frontend/main.tf +++ b/tf/modules/ooniapi_frontend/main.tf @@ -261,378 +261,151 @@ resource "aws_alb_listener_rule" "ooniapi_th" { tags = var.tags } -resource "aws_lb_listener_rule" "ooniapi_ooniauth_rule" { - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 108 +locals { + # routes.yaml is shared with the gateway on the Hetzner hosts + # (ansible/roles/ooniapi_gateway), so both serve the same routes + routes = yamldecode(file("${path.module}/routes.yaml")) - action { - type = "forward" - target_group_arn = var.ooniapi_ooniauth_target_group_arn + target_groups = { + ooniauth = var.ooniapi_ooniauth_target_group_arn + oonirun = var.ooniapi_oonirun_target_group_arn + ooniprobe = var.ooniapi_ooniprobe_target_group_arn + ooniprobe_legacy = var.ooniapi_ooniprobe_legacy_target_group_arn + oonifindings = var.ooniapi_oonifindings_target_group_arn + oonimeasurements = var.ooniapi_oonimeasurements_target_group_arn + testlists = var.ooniapi_testlists_target_group_arn } - condition { - path_pattern { - values = [ - "/api/v2/ooniauth/*", - "/api/v1/user_register", - "/api/v1/user_login", - "/api/v1/user_refresh_token", - "/api/_/account_metadata", - ] - } - } + listener_rules = { for r in local.routes : r.name => r } } -resource "aws_lb_listener_rule" "ooniapi_ooniauth_rule_host" { - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 109 - - action { - type = "forward" - target_group_arn = var.ooniapi_ooniauth_target_group_arn - } - - condition { - host_header { - values = ["ooniauth.${local.direct_domain_suffix}"] - } - } -} +resource "aws_lb_listener_rule" "route" { + for_each = local.listener_rules -resource "aws_lb_listener_rule" "ooniapi_oonirun_rule" { listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 110 + priority = each.value.priority action { type = "forward" - target_group_arn = var.ooniapi_oonirun_target_group_arn + target_group_arn = local.target_groups[each.value.service] } - condition { - path_pattern { - values = ["/api/v2/oonirun/*"] + dynamic "condition" { + for_each = can(each.value.paths) ? [each.value.paths] : [] + content { + path_pattern { + values = condition.value + } } - } -} -resource "aws_lb_listener_rule" "ooniapi_oonirun_rule_host" { - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 111 - - action { - type = "forward" - target_group_arn = var.ooniapi_oonirun_target_group_arn - } - - condition { - host_header { - values = ["oonirun.${local.direct_domain_suffix}"] + dynamic "condition" { + for_each = try(each.value.direct_host, false) ? [each.value.service] : [] + content { + host_header { + values = ["${condition.value}.${local.direct_domain_suffix}"] + } } } -} - -resource "aws_lb_listener_rule" "ooniapi_ooniprobe_rule" { - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 120 - - action { - type = "forward" - target_group_arn = var.ooniapi_ooniprobe_target_group_arn - } - - condition { - path_pattern { - values = [ - "/api/v2/ooniprobe/*", - "/api/v1/login", - "/api/v1/register", - "/api/v1/update/*", - "/api/v1/check-in*" - ] + dynamic "condition" { + for_each = can(each.value.http_header) ? [each.value.http_header] : [] + content { + http_header { + http_header_name = condition.value.name + values = condition.value.values + } } } } -resource "aws_lb_listener_rule" "ooniapi_ooniprobe_rule_2" { - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 121 - - action { - type = "forward" - target_group_arn = var.ooniapi_ooniprobe_target_group_arn - } - - condition { - path_pattern { - values = [ - "/api/v1/test-helpers*", - "/api/v1/test-list/urls", - "/report*", - "/api/_/show_countries_prioritization", - "/api/_/debug_prioritization" - ] - } - } +# the rules were one resource each before routes.yaml +moved { + from = aws_lb_listener_rule.ooniapi_ooniauth_rule + to = aws_lb_listener_rule.route["ooniauth_rule"] } -resource "aws_lb_listener_rule" "ooniapi_ooniprobe_rule_3_legacy_version" { - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 122 - - action { - type = "forward" - target_group_arn = var.ooniapi_ooniprobe_legacy_target_group_arn - } - - condition { - path_pattern { - values = [ - "/api/v1/manifest*", - "/api/v1/submit_measurement*", - "/api/v1/sign_credential*" - ] - } - } - - condition { - http_header { - http_header_name = "X-Protocol-Version" - values = ["0.1.0"] - } - } +moved { + from = aws_lb_listener_rule.ooniapi_ooniauth_rule_host + to = aws_lb_listener_rule.route["ooniauth_rule_host"] } -resource "aws_lb_listener_rule" "ooniapi_ooniprobe_rule_3_current_version" { - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 124 - - action { - type = "forward" - target_group_arn = var.ooniapi_ooniprobe_target_group_arn - } - - condition { - path_pattern { - values = [ - "/api/v1/manifest*", - "/api/v1/submit_measurement*", - "/api/v1/sign_credential*" - ] - } - } - - # matches any value, but only when the header is present - condition { - http_header { - http_header_name = "X-Protocol-Version" - values = ["*"] - } - } +moved { + from = aws_lb_listener_rule.ooniapi_oonirun_rule + to = aws_lb_listener_rule.route["oonirun_rule"] } -resource "aws_lb_listener_rule" "ooniapi_ooniprobe_rule_3_no_version" { - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 126 - - action { - type = "forward" - target_group_arn = var.ooniapi_ooniprobe_legacy_target_group_arn - } - - # No X-Protocol-Version header at all: rules 122/124 above already - # matched every request that does carry the header, so anything left - # here is header-less and should go to the legacy service. - condition { - path_pattern { - values = [ - "/api/v1/manifest*", - "/api/v1/submit_measurement*", - "/api/v1/sign_credential*" - ] - } - } +moved { + from = aws_lb_listener_rule.ooniapi_oonirun_rule_host + to = aws_lb_listener_rule.route["oonirun_rule_host"] } -resource "aws_lb_listener_rule" "ooniapi_ooniprobe_rule_4" { - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 123 - - action { - type = "forward" - target_group_arn = var.ooniapi_ooniprobe_target_group_arn - } - - condition { - path_pattern { - values = [ - "/bouncer/net-tests*", - "/api/v1/geolookup*", - "/api/v1/collectors*", - "/api/v1/test-list/tor-targets", - "/api/v1/test-list/psiphon-config" - ] - } - } +moved { + from = aws_lb_listener_rule.ooniapi_ooniprobe_rule + to = aws_lb_listener_rule.route["ooniprobe_rule"] } -resource "aws_lb_listener_rule" "ooniapi_ooniprobe_rule_host" { - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 125 - - action { - type = "forward" - target_group_arn = var.ooniapi_ooniprobe_target_group_arn - } - - - condition { - host_header { - values = ["ooniprobe.${local.direct_domain_suffix}"] - } - } - +moved { + from = aws_lb_listener_rule.ooniapi_ooniprobe_rule_2 + to = aws_lb_listener_rule.route["ooniprobe_rule_2"] } -resource "aws_lb_listener_rule" "ooniapi_oonifindings_rule" { - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 130 - - action { - type = "forward" - target_group_arn = var.ooniapi_oonifindings_target_group_arn - } - - condition { - path_pattern { - values = [ - "/api/v1/incidents/*", - ] - } - } +moved { + from = aws_lb_listener_rule.ooniapi_ooniprobe_rule_3_legacy_version + to = aws_lb_listener_rule.route["ooniprobe_rule_3_legacy_version"] } -resource "aws_lb_listener_rule" "ooniapi_oonifindings_rule_host" { - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 131 - - action { - type = "forward" - target_group_arn = var.ooniapi_oonifindings_target_group_arn - } - condition { - host_header { - values = ["oonifindings.${local.direct_domain_suffix}"] - } - } +moved { + from = aws_lb_listener_rule.ooniapi_ooniprobe_rule_4 + to = aws_lb_listener_rule.route["ooniprobe_rule_4"] } -resource "aws_lb_listener_rule" "ooniapi_oonimeasurements_rule_host" { - # hotfix: to allow us to deploy the frontend without the measurements service - count = var.ooniapi_oonimeasurements_target_group_arn != null ? 1 : 0 - - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 139 - - action { - type = "forward" - target_group_arn = var.ooniapi_oonimeasurements_target_group_arn - } - condition { - host_header { - values = ["oonimeasurements.${local.direct_domain_suffix}"] - } - } +moved { + from = aws_lb_listener_rule.ooniapi_ooniprobe_rule_3_current_version + to = aws_lb_listener_rule.route["ooniprobe_rule_3_current_version"] } -resource "aws_lb_listener_rule" "ooniapi_oonimeasurements_rule_1" { - # hotfix: to allow us to deploy the frontend without the measurements service - count = var.ooniapi_oonimeasurements_target_group_arn != null ? 1 : 0 - - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 140 - - action { - type = "forward" - target_group_arn = var.ooniapi_oonimeasurements_target_group_arn - } - - condition { - path_pattern { - values = [ - "/api/v1/measurements/*", - "/api/v1/raw_measurement", - "/api/v1/measurement_meta", - "/api/v1/measurements", - "/api/v1/torsf_stats" - ] - } - } +moved { + from = aws_lb_listener_rule.ooniapi_ooniprobe_rule_host + to = aws_lb_listener_rule.route["ooniprobe_rule_host"] } -resource "aws_lb_listener_rule" "ooniapi_oonimeasurements_rule_2" { - # hotfix: to allow us to deploy the frontend without the measurements service - count = var.ooniapi_oonimeasurements_target_group_arn != null ? 1 : 0 - - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 142 - - action { - type = "forward" - target_group_arn = var.ooniapi_oonimeasurements_target_group_arn - } +moved { + from = aws_lb_listener_rule.ooniapi_ooniprobe_rule_3_no_version + to = aws_lb_listener_rule.route["ooniprobe_rule_3_no_version"] +} - condition { - path_pattern { - values = [ - "/api/v1/aggregation", - "/api/v1/aggregation/*", - "/api/v1/observations", - "/api/v1/analysis", - ] - } - } +moved { + from = aws_lb_listener_rule.ooniapi_oonifindings_rule + to = aws_lb_listener_rule.route["oonifindings_rule"] } -resource "aws_lb_listener_rule" "ooniapi_oonimeasurements_rule_3" { +moved { + from = aws_lb_listener_rule.ooniapi_oonifindings_rule_host + to = aws_lb_listener_rule.route["oonifindings_rule_host"] +} - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 143 +moved { + from = aws_lb_listener_rule.ooniapi_oonimeasurements_rule_host[0] + to = aws_lb_listener_rule.route["oonimeasurements_rule_host"] +} - action { - type = "forward" - target_group_arn = var.ooniapi_oonimeasurements_target_group_arn - } +moved { + from = aws_lb_listener_rule.ooniapi_oonimeasurements_rule_1[0] + to = aws_lb_listener_rule.route["oonimeasurements_rule_1"] +} - condition { - path_pattern { - values = [ - "/api/v1/detector/changepoints", - ] - } - } +moved { + from = aws_lb_listener_rule.ooniapi_oonimeasurements_rule_2[0] + to = aws_lb_listener_rule.route["oonimeasurements_rule_2"] } -resource "aws_lb_listener_rule" "ooniapi_testlists_rule" { - # hotfix: to allow us to deploy the frontend without the testlists service - count = var.ooniapi_testlists_target_group_arn != null ? 1 : 0 - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 144 +moved { + from = aws_lb_listener_rule.ooniapi_oonimeasurements_rule_3 + to = aws_lb_listener_rule.route["oonimeasurements_rule_3"] +} - action { - type = "forward" - target_group_arn = var.ooniapi_testlists_target_group_arn - } - condition { - path_pattern { - values = [ - "/api/_/url-submission/test-list/*", - "/api/_/url-priorities/list", - "/api/_/url-priorities/update", - "/api/v1/url-submission/submit", - "/api/v1/url-submission/update-url", - ] - } - } +moved { + from = aws_lb_listener_rule.ooniapi_testlists_rule[0] + to = aws_lb_listener_rule.route["testlists_rule"] } diff --git a/tf/modules/ooniapi_frontend/routes.yaml b/tf/modules/ooniapi_frontend/routes.yaml new file mode 100644 index 00000000..b32462df --- /dev/null +++ b/tf/modules/ooniapi_frontend/routes.yaml @@ -0,0 +1,156 @@ +# Routes of the OONI API, the one place they are declared. Read by: +# - Terraform, tf/modules/ooniapi_frontend: one ALB listener rule per entry +# (aws_lb_listener_rule.route[""]), for the services on ECS. +# Anything no route matches goes to reverseproxy. +# +# Each entry is one ALB rule, so the ALB's limits apply: at most 5 values +# over all its conditions. Fields: +# name resource key of the rule; keep stable, renaming one makes +# Terraform replace the rule +# priority ALB rule priority, lowest matches first +# service target service (ALB target group) +# paths ALB path patterns: exact, or ending in * for a prefix +# direct_host route . to the service +# http_header only match requests with this header and one of these values + +- name: ooniauth_rule + priority: 108 + service: ooniauth + paths: + - "/api/v2/ooniauth/*" + - "/api/v1/user_register" + - "/api/v1/user_login" + - "/api/v1/user_refresh_token" + - "/api/_/account_metadata" + +- name: ooniauth_rule_host + priority: 109 + service: ooniauth + direct_host: true + +- name: oonirun_rule + priority: 110 + service: oonirun + paths: + - "/api/v2/oonirun/*" + +- name: oonirun_rule_host + priority: 111 + service: oonirun + direct_host: true + +- name: ooniprobe_rule + priority: 120 + service: ooniprobe + paths: + - "/api/v2/ooniprobe/*" + - "/api/v1/login" + - "/api/v1/register" + - "/api/v1/update/*" + - "/api/v1/check-in*" + +- name: ooniprobe_rule_2 + priority: 121 + service: ooniprobe + paths: + - "/api/v1/test-helpers*" + - "/api/v1/test-list/urls" + - "/report*" + - "/api/_/show_countries_prioritization" + - "/api/_/debug_prioritization" + +- name: ooniprobe_rule_3_legacy_version + priority: 122 + service: ooniprobe_legacy + paths: + - "/api/v1/manifest*" + - "/api/v1/submit_measurement*" + - "/api/v1/sign_credential*" + http_header: + name: X-Protocol-Version + values: ["0.1.0"] + +- name: ooniprobe_rule_4 + priority: 123 + service: ooniprobe + paths: + - "/bouncer/net-tests*" + - "/api/v1/geolookup*" + - "/api/v1/collectors*" + - "/api/v1/test-list/tor-targets" + - "/api/v1/test-list/psiphon-config" + +- name: ooniprobe_rule_3_current_version + priority: 124 + service: ooniprobe + paths: + - "/api/v1/manifest*" + - "/api/v1/submit_measurement*" + - "/api/v1/sign_credential*" + http_header: + name: X-Protocol-Version + values: ["*"] + +- name: ooniprobe_rule_host + priority: 125 + service: ooniprobe + direct_host: true + +- name: ooniprobe_rule_3_no_version + priority: 126 + service: ooniprobe_legacy + paths: + - "/api/v1/manifest*" + - "/api/v1/submit_measurement*" + - "/api/v1/sign_credential*" + +- name: oonifindings_rule + priority: 130 + service: oonifindings + paths: + - "/api/v1/incidents/*" + +- name: oonifindings_rule_host + priority: 131 + service: oonifindings + direct_host: true + +- name: oonimeasurements_rule_host + priority: 139 + service: oonimeasurements + direct_host: true + +- name: oonimeasurements_rule_1 + priority: 140 + service: oonimeasurements + paths: + - "/api/v1/measurements/*" + - "/api/v1/raw_measurement" + - "/api/v1/measurement_meta" + - "/api/v1/measurements" + - "/api/v1/torsf_stats" + +- name: oonimeasurements_rule_2 + priority: 142 + service: oonimeasurements + paths: + - "/api/v1/aggregation" + - "/api/v1/aggregation/*" + - "/api/v1/observations" + - "/api/v1/analysis" + +- name: oonimeasurements_rule_3 + priority: 143 + service: oonimeasurements + paths: + - "/api/v1/detector/changepoints" + +- name: testlists_rule + priority: 144 + service: testlists + paths: + - "/api/_/url-submission/test-list/*" + - "/api/_/url-priorities/list" + - "/api/_/url-priorities/update" + - "/api/v1/url-submission/submit" + - "/api/v1/url-submission/update-url" diff --git a/tf/modules/ooniapi_frontend/variables.tf b/tf/modules/ooniapi_frontend/variables.tf index 614056e6..47ff4f10 100644 --- a/tf/modules/ooniapi_frontend/variables.tf +++ b/tf/modules/ooniapi_frontend/variables.tf @@ -38,12 +38,12 @@ variable "ooniapi_oonifindings_target_group_arn" { variable "ooniapi_oonimeasurements_target_group_arn" { description = "arn for the target group of the oonimeasurements service" - default = null + nullable = false } variable "ooniapi_testlists_target_group_arn" { description = "arn for the target group of the testlists service" - default = null + nullable = false } variable "dns_zone_ooni_io" { From fbb91b852e0be63585cc370061eabceb7a3a0bd3 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Sat, 3 Oct 2026 01:21:25 +0000 Subject: [PATCH 11/24] Generate the gateway's nginx locations from routes.yaml The gateway's locations were a hand written copy of the ALB rules, and had already fallen behind them (main's X-Protocol-Version rules). They are now generated from tf/modules/ooniapi_frontend/routes.yaml, which the ALB rules are generated from too, as are the per service direct-host vhosts. nginx can't follow ALB priorities: it picks an exact match, then the longest prefix. filter_plugins/ooniapi_routes.py turns each ALB pattern into an exact or ^~ prefix location and lists the routes nginx couldn't serve the same way (a wildcard other than a trailing *, the same path twice, a prefix of one service covering a path of another); the role fails on any. A plugin rather than inline Jinja, so the result doesn't depend on how a given ansible-core converts templated strings. Routes marked legacy (ooniprobe_legacy, being retired) are left to the ALB: the gateway doesn't run it, so those paths go to ooniprobe. Rendered with ansible-core 2.21 for all six services plus testlists proxied cross-cloud, the generated config has the same 46 locations, with the same match types and upstreams, as the hand written one. --- .../roles/ooniapi_gateway/defaults/main.yml | 4 + .../filter_plugins/ooniapi_routes.py | 63 +++++++++++++ ansible/roles/ooniapi_gateway/tasks/main.yml | 3 + .../roles/ooniapi_gateway/tasks/routes.yml | 14 +++ .../ooniapi_gateway/templates/gateway.conf.j2 | 89 ++++--------------- tf/modules/ooniapi_frontend/routes.yaml | 13 ++- 6 files changed, 112 insertions(+), 74 deletions(-) create mode 100644 ansible/roles/ooniapi_gateway/filter_plugins/ooniapi_routes.py create mode 100644 ansible/roles/ooniapi_gateway/tasks/routes.yml diff --git a/ansible/roles/ooniapi_gateway/defaults/main.yml b/ansible/roles/ooniapi_gateway/defaults/main.yml index 4f910dc7..4d5d50b8 100644 --- a/ansible/roles/ooniapi_gateway/defaults/main.yml +++ b/ansible/roles/ooniapi_gateway/defaults/main.yml @@ -57,6 +57,10 @@ ooniapi_gateway_testlists_upstream: "" # Docker network shared by every service's containers on this host. ooniapi_gateway_network_name: ooniapi +# Routes of the API, shared with the ALB (see the file's header); the +# gateway's locations are generated from them in tasks/main.yml +ooniapi_gateway_routes: "{{ lookup('ansible.builtin.file', role_path ~ '/../../../tf/modules/ooniapi_frontend/routes.yaml') | from_yaml }}" + # SSH user the deploy CodeBuild job connects as. Must match # `deploy_ssh_user` (default "deploy") in the ooniapi_service_deployer # Terraform module. diff --git a/ansible/roles/ooniapi_gateway/filter_plugins/ooniapi_routes.py b/ansible/roles/ooniapi_gateway/filter_plugins/ooniapi_routes.py new file mode 100644 index 00000000..61c0e0c1 --- /dev/null +++ b/ansible/roles/ooniapi_gateway/filter_plugins/ooniapi_routes.py @@ -0,0 +1,63 @@ +# Filters turning tf/modules/ooniapi_frontend/routes.yaml (shared with the +# ALB) into the gateway's nginx locations; see tasks/main.yml. +# +# nginx picks a location by exact match, then longest prefix, where the ALB +# goes by rule priority. The two agree as long as every ALB pattern is exact +# or a prefix (a trailing *), and no prefix of one service covers a path of +# another; ooniapi_gateway_route_problems lists the routes where they would +# not. + + +def _locations(routes): + out = [] + for r in sorted(routes, key=lambda r: r["priority"]): + # the ALB sends these to ooniprobe_legacy, which the gateway doesn't + # run: they go to ooniprobe, whatever their X-Protocol-Version + if r.get("legacy"): + continue + for p in r.get("paths", []): + prefix = p.endswith("*") + out.append({ + "route": r["name"], + "service": r["service"], + "match": "^~" if prefix else "=", + "path": p[:-1] if prefix else p, + }) + return out + + +def ooniapi_gateway_locations(routes): + return _locations(routes) + + +def ooniapi_gateway_direct_hosts(routes): + return list(dict.fromkeys(r["service"] for r in routes if r.get("direct_host"))) + + +def ooniapi_gateway_route_problems(routes): + locations = _locations(routes) + problems = [] + seen = {} + for loc in locations: + if "*" in loc["path"] or "?" in loc["path"]: + problems.append(f"{loc['route']}: {loc['path']}: only exact paths and a trailing * can be served by nginx") + key = (loc["match"], loc["path"]) + if key in seen: + problems.append(f"{loc['route']}: {loc['path']} is also routed by {seen[key]}") + seen.setdefault(key, loc["route"]) + for a in locations: + if a["match"] != "^~": + continue + for b in locations: + if b["service"] != a["service"] and b["path"].startswith(a["path"]): + problems.append(f"{a['route']}: {a['path']}* covers {b['route']}'s {b['path']}, nginx would not follow ALB priorities") + return problems + + +class FilterModule: + def filters(self): + return { + "ooniapi_gateway_locations": ooniapi_gateway_locations, + "ooniapi_gateway_direct_hosts": ooniapi_gateway_direct_hosts, + "ooniapi_gateway_route_problems": ooniapi_gateway_route_problems, + } diff --git a/ansible/roles/ooniapi_gateway/tasks/main.yml b/ansible/roles/ooniapi_gateway/tasks/main.yml index 2a9ac98a..7fc36e8b 100644 --- a/ansible/roles/ooniapi_gateway/tasks/main.yml +++ b/ansible/roles/ooniapi_gateway/tasks/main.yml @@ -217,6 +217,9 @@ mode: "0644" notify: reload ooniapi gateway nginx +- name: Derive and check the gateway's locations + ansible.builtin.import_tasks: routes.yml + - name: Render the gateway vhost ansible.builtin.template: src: gateway.conf.j2 diff --git a/ansible/roles/ooniapi_gateway/tasks/routes.yml b/ansible/roles/ooniapi_gateway/tasks/routes.yml new file mode 100644 index 00000000..a6478c7c --- /dev/null +++ b/ansible/roles/ooniapi_gateway/tasks/routes.yml @@ -0,0 +1,14 @@ +--- +# The gateway's locations are generated from routes.yaml, shared with the +# ALB, by filter_plugins/ooniapi_routes.py, which also checks nginx can +# serve them the way the ALB does. +- name: Derive the gateway's locations from routes.yaml + ansible.builtin.set_fact: + ooniapi_gateway_locations: "{{ ooniapi_gateway_routes | ooniapi_gateway_locations }}" + ooniapi_gateway_direct_hosts: "{{ ooniapi_gateway_routes | ooniapi_gateway_direct_hosts }}" + ooniapi_gateway_route_problems: "{{ ooniapi_gateway_routes | ooniapi_gateway_route_problems }}" + +- name: Check that nginx can serve routes.yaml like the ALB does + ansible.builtin.assert: + that: ooniapi_gateway_route_problems | length == 0 + fail_msg: "{{ ooniapi_gateway_route_problems }}" diff --git a/ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 b/ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 index e87e46ef..b7a32a92 100644 --- a/ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 +++ b/ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 @@ -1,11 +1,12 @@ # Managed by ansible - roles/ooniapi_gateway/templates/gateway.conf.j2 # -# Single "gateway" vhost for the OONI tier0 API on this dedicated host, -# mirroring (1:1, by design) the AWS ALB listener rules in -# tf/modules/ooniapi_frontend/main.tf. Only rules for services actually -# running blue/green here ({{ ooniapi_gateway_services | map(attribute='name') | join(', ') }}) -# are handled; everything unmatched falls through to "reverseproxy", exactly -# like the ALB's default_action. +# Single "gateway" vhost for the OONI tier0 API on this dedicated host. Its +# locations are generated from tf/modules/ooniapi_frontend/routes.yaml, which +# the AWS ALB's listener rules are generated from too, so both route alike. +# Only routes of services running blue/green here +# ({{ ooniapi_gateway_services | map(attribute='name') | join(', ') }}) and of +# those proxied cross-cloud are handled; everything unmatched falls through +# to "reverseproxy", exactly like the ALB's default_action. # # Each service's own `-upstream.conf` (the `upstream { # ... }` block referenced below via proxy_pass) is dropped into this same @@ -63,70 +64,16 @@ server { proxy_set_header X-Forwarded-Proto $scheme; proxy_read_timeout 900; -{% if 'ooniauth' in service_names %} - ## --- ooniauth --- (ALB priority 108) - location ^~ /api/v2/ooniauth/ { proxy_pass http://ooniauth; } - location = /api/v1/user_register { proxy_pass http://ooniauth; } - location = /api/v1/user_login { proxy_pass http://ooniauth; } - location = /api/v1/user_refresh_token { proxy_pass http://ooniauth; } - location = /api/_/account_metadata { proxy_pass http://ooniauth; } -{% endif %} - -{% if 'oonirun' in service_names %} - ## --- oonirun --- (ALB priority 110) - location ^~ /api/v2/oonirun/ { proxy_pass http://oonirun; } -{% endif %} - -{% if 'ooniprobe' in service_names %} - ## --- ooniprobe --- (ALB priorities 120-123) - location ^~ /api/v2/ooniprobe/ { proxy_pass http://ooniprobe; } - location = /api/v1/login { proxy_pass http://ooniprobe; } - location = /api/v1/register { proxy_pass http://ooniprobe; } - location ^~ /api/v1/update/ { proxy_pass http://ooniprobe; } - location ^~ /api/v1/check-in { proxy_pass http://ooniprobe; } - location ^~ /api/v1/test-helpers { proxy_pass http://ooniprobe; } - location = /api/v1/test-list/urls { proxy_pass http://ooniprobe; } - location ^~ /report { proxy_pass http://ooniprobe; } - location = /api/_/show_countries_prioritization { proxy_pass http://ooniprobe; } - location = /api/_/debug_prioritization { proxy_pass http://ooniprobe; } - location ^~ /api/v1/manifest { proxy_pass http://ooniprobe; } - location ^~ /api/v1/sign_credential { proxy_pass http://ooniprobe; } - location ^~ /api/v1/submit_measurement { proxy_pass http://ooniprobe; } - location ^~ /bouncer/net-tests { proxy_pass http://ooniprobe; } - location ^~ /api/v1/geolookup { proxy_pass http://ooniprobe; } - location ^~ /api/v1/collectors { proxy_pass http://ooniprobe; } - location = /api/v1/test-list/tor-targets { proxy_pass http://ooniprobe; } - location = /api/v1/test-list/psiphon-config { proxy_pass http://ooniprobe; } -{% endif %} - -{% if 'oonifindings' in service_names %} - ## --- oonifindings --- (ALB priority 130) - location ^~ /api/v1/incidents/ { proxy_pass http://oonifindings; } -{% endif %} - -{% if 'oonimeasurements' in service_names %} - ## --- oonimeasurements --- (ALB priorities 140-143) - location ^~ /api/v1/measurements/ { proxy_pass http://oonimeasurements; } - location = /api/v1/raw_measurement { proxy_pass http://oonimeasurements; } - location = /api/v1/measurement_meta { proxy_pass http://oonimeasurements; } - location = /api/v1/measurements { proxy_pass http://oonimeasurements; } - location = /api/v1/torsf_stats { proxy_pass http://oonimeasurements; } - location = /api/v1/aggregation { proxy_pass http://oonimeasurements; } - location ^~ /api/v1/aggregation/ { proxy_pass http://oonimeasurements; } - location = /api/v1/observations { proxy_pass http://oonimeasurements; } - location = /api/v1/analysis { proxy_pass http://oonimeasurements; } - location = /api/v1/detector/changepoints { proxy_pass http://oonimeasurements; } -{% endif %} - -{% if ooniapi_gateway_testlists_upstream %} - ## --- testlists (ALB priority 144; not yet migrated off AWS, proxied - ## cross-cloud until it is) --- - location ^~ /api/_/url-submission/test-list/ { proxy_pass https://{{ ooniapi_gateway_testlists_upstream }}; } - location = /api/_/url-priorities/list { proxy_pass https://{{ ooniapi_gateway_testlists_upstream }}; } - location = /api/_/url-priorities/update { proxy_pass https://{{ ooniapi_gateway_testlists_upstream }}; } - location = /api/v1/url-submission/submit { proxy_pass https://{{ ooniapi_gateway_testlists_upstream }}; } - location = /api/v1/url-submission/update-url { proxy_pass https://{{ ooniapi_gateway_testlists_upstream }}; } +{% set remote_upstreams = {'testlists': 'https://' ~ ooniapi_gateway_testlists_upstream} if ooniapi_gateway_testlists_upstream else {} %} + ## --- generated from routes.yaml, see tasks/main.yml --- +{% for l in ooniapi_gateway_locations %} +{% if l.service in service_names %} + location {{ l.match }} {{ l.path }} { proxy_pass http://{{ l.service }}; } # {{ l.route }} +{% elif l.service in remote_upstreams %} + # {{ l.service }} is not yet migrated off AWS, proxied cross-cloud until it is + location {{ l.match }} {{ l.path }} { proxy_pass {{ remote_upstreams[l.service] }}; } # {{ l.route }} {% endif %} +{% endfor %} ## --- default: everything else goes to reverseproxy, same as the ALB's ## default_action. Covers legacy/unmigrated paths (oonith is NOT among @@ -137,8 +84,7 @@ server { } } -{% for service in service_names %} -{% if service != 'reverseproxy' %} +{% for service in ooniapi_gateway_direct_hosts if service in service_names %} server { listen 443 ssl http2; listen [::]:443 ssl http2; @@ -163,5 +109,4 @@ server { proxy_pass http://{{ service }}; } } -{% endif %} {% endfor %} diff --git a/tf/modules/ooniapi_frontend/routes.yaml b/tf/modules/ooniapi_frontend/routes.yaml index b32462df..3a0bb8fe 100644 --- a/tf/modules/ooniapi_frontend/routes.yaml +++ b/tf/modules/ooniapi_frontend/routes.yaml @@ -1,17 +1,24 @@ # Routes of the OONI API, the one place they are declared. Read by: # - Terraform, tf/modules/ooniapi_frontend: one ALB listener rule per entry # (aws_lb_listener_rule.route[""]), for the services on ECS. -# Anything no route matches goes to reverseproxy. +# - Ansible, roles/ooniapi_gateway: the nginx locations of the gateway on the +# Hetzner hosts, for the services deployed there blue/green. +# Anything no route matches goes to reverseproxy, on both. # # Each entry is one ALB rule, so the ALB's limits apply: at most 5 values # over all its conditions. Fields: # name resource key of the rule; keep stable, renaming one makes # Terraform replace the rule -# priority ALB rule priority, lowest matches first +# priority ALB rule priority, lowest matches first. The gateway can't +# order its locations by it, so a prefix of one service must +# not cover a path of another (checked by the gateway role) # service target service (ALB target group) # paths ALB path patterns: exact, or ending in * for a prefix # direct_host route . to the service # http_header only match requests with this header and one of these values +# (ALB only: they only choose between ooniprobe and its legacy) +# legacy ALB only: ooniprobe_legacy is being retired and does not run +# on the gateway, which sends these paths to ooniprobe - name: ooniauth_rule priority: 108 @@ -69,6 +76,7 @@ http_header: name: X-Protocol-Version values: ["0.1.0"] + legacy: true - name: ooniprobe_rule_4 priority: 123 @@ -103,6 +111,7 @@ - "/api/v1/manifest*" - "/api/v1/submit_measurement*" - "/api/v1/sign_credential*" + legacy: true - name: oonifindings_rule priority: 130 From e8a184ee2fa27f02d4a6e7e16cc5e1049844e5d2 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Sat, 3 Oct 2026 01:23:28 +0000 Subject: [PATCH 12/24] ooniapi_gateway: use a syslog tag nginx accepts nginx only allows letters, digits and underscores in a syslog tag, so nginx -t rejected the gateway vhost ("syslog \"tag\" only allows alphanumeric characters and underscore", nginx 1.27) and it could never have been installed. The tag is now ooniapi_gateway. --- ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 b/ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 index b7a32a92..76782f9e 100644 --- a/ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 +++ b/ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 @@ -32,8 +32,8 @@ server { listen [::]:443 ssl http2; server_name {{ ([ooniapi_gateway_primary_domain] + ooniapi_gateway_extra_server_names) | join(' ') }}; - access_log syslog:server=unix:/dev/log,tag=ooniapi-gateway,severity=info ooniapi_gateway_fmt; - error_log syslog:server=unix:/dev/log,tag=ooniapi-gateway,severity=info; + access_log syslog:server=unix:/dev/log,tag=ooniapi_gateway,severity=info ooniapi_gateway_fmt; + error_log syslog:server=unix:/dev/log,tag=ooniapi_gateway,severity=info; client_max_body_size 200M; # for measurement POST, matches the legacy API vhost @@ -90,8 +90,8 @@ server { listen [::]:443 ssl http2; server_name {{ service }}.{{ ooniapi_gateway_direct_domain_suffix }}; - access_log syslog:server=unix:/dev/log,tag=ooniapi-gateway,severity=info ooniapi_gateway_fmt; - error_log syslog:server=unix:/dev/log,tag=ooniapi-gateway,severity=info; + access_log syslog:server=unix:/dev/log,tag=ooniapi_gateway,severity=info ooniapi_gateway_fmt; + error_log syslog:server=unix:/dev/log,tag=ooniapi_gateway,severity=info; ssl_certificate {{ ooniapi_gateway_cert_path }}{{ ooniapi_gateway_cert_name }}/fullchain.pem; ssl_certificate_key {{ ooniapi_gateway_cert_path }}{{ ooniapi_gateway_cert_name }}/privkey.pem; From ac8df6d6a23b63045a170e9b9204b5a472bf81ff Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Sat, 3 Oct 2026 01:23:39 +0000 Subject: [PATCH 13/24] ooniapi_gateway: cache responses of the read heavy routes Nothing caches oonimeasurements or oonirun today: on api.ooni.org every repeated request reaches uvicorn (only the reverseproxy's /api/_/ paths show x-cache-status: HIT), and for oonimeasurements that means ClickHouse, saturated on data2. The gateway now caches GET/HEAD responses of the routes routes.yaml marks cache (oonimeasurements, oonirun), as long as their Cache-Control allows, or 10 s without one, like the legacy ooni-api vhost's apicache. Requests with an Authorization header never use the cache (oonirun links can differ per user), and nginx doesn't store no-cache, private or max-age=0 responses, or any setting a cookie. Two zones, since nginx keeps an entry until it goes unused for inactive, expired or not: lists and aggregations valid for minutes would otherwise push out measurement bodies valid for a day. Sized per host as if it got all the traffic, from data2's query_log and responses of api.ooni.org: - long (raw_measurement, measurement_meta; valid 1 d): at most ~50K distinct bodies a day of ~76 KB (max seen ~160 KB), ~3.8 GB a day: max_size 8g, keys_zone 16m (~130K keys), inactive 1d. - short (everything else cached; valid 10 s to 1 h): ~7.5K requests an hour of up to ~60 KB, ~450 MB an hour if all were distinct: max_size 2g, keys_zone 8m, inactive 1h. The access log now records $upstream_cache_status, to tune these from real hit rates. Cache directives and X-Cache-Status are set per server, since an add_header in a location would drop the server's HSTS headers. Checked with nginx 1.27 and stub upstreams: lists, bodies (long zone) and header-less aggregations miss then hit; Authorization bypasses and isn't stored; no-cache isn't stored; ooniprobe and the default route aren't cached; every response keeps HSTS. --- .../roles/ooniapi_gateway/defaults/main.yml | 21 +++++++++++++++ .../filter_plugins/ooniapi_routes.py | 10 +++++++ ansible/roles/ooniapi_gateway/tasks/main.yml | 9 +++++++ .../ooniapi_gateway/templates/gateway.conf.j2 | 27 +++++++++++++++++-- tf/modules/ooniapi_frontend/routes.yaml | 11 ++++++++ 5 files changed, 76 insertions(+), 2 deletions(-) diff --git a/ansible/roles/ooniapi_gateway/defaults/main.yml b/ansible/roles/ooniapi_gateway/defaults/main.yml index 4d5d50b8..6a78c903 100644 --- a/ansible/roles/ooniapi_gateway/defaults/main.yml +++ b/ansible/roles/ooniapi_gateway/defaults/main.yml @@ -61,6 +61,27 @@ ooniapi_gateway_network_name: ooniapi # gateway's locations are generated from them in tasks/main.yml ooniapi_gateway_routes: "{{ lookup('ansible.builtin.file', role_path ~ '/../../../tf/modules/ooniapi_frontend/routes.yaml') | from_yaml }}" +# Response caches for the routes marked cache/cache_long in routes.yaml. +# nginx keeps an entry until it goes unused for `inactive`, expired or not, +# so responses valid for minutes and measurements valid for a day get zones +# of their own, and the first can't push the second out. Sized per host, as +# if it got all the traffic (2026-10, data2 query_log and api.ooni.org): +# - long: measurement bodies (raw_measurement, measurement_meta), valid 1 d: +# at most ~50K distinct a day (28K measurement_meta + 25K raw lookups) of +# ~76 KB (max seen ~160 KB), ~3.8 GB a day; 2x that, ~130K keys. +# - short: lists, aggregations, oonirun (valid 10 s to 1 h): ~7.5K requests +# an hour of up to ~60 KB (aggregations 0.3-22 KB, lists 35-70 KB, +# observations ~135 KB), ~450 MB an hour if all were distinct; 4x that. +# 1 MB of keys_zone holds ~8K keys. nginx stores responses uncompressed. +ooniapi_gateway_cache_dir: /var/cache/nginx/ooniapi-gateway +ooniapi_gateway_cache_zones: + short: {keys_zone: 8m, max_size: 2g, inactive: 1h} + long: {keys_zone: 16m, max_size: 8g, inactive: 1d} +# for responses without Cache-Control, like /api/v1/aggregation's today; +# the same as the legacy ooni-api vhost's apicache +ooniapi_gateway_cache_default_valid: 10s +ooniapi_gateway_nginx_user: www-data + # SSH user the deploy CodeBuild job connects as. Must match # `deploy_ssh_user` (default "deploy") in the ooniapi_service_deployer # Terraform module. diff --git a/ansible/roles/ooniapi_gateway/filter_plugins/ooniapi_routes.py b/ansible/roles/ooniapi_gateway/filter_plugins/ooniapi_routes.py index 61c0e0c1..1a17b46d 100644 --- a/ansible/roles/ooniapi_gateway/filter_plugins/ooniapi_routes.py +++ b/ansible/roles/ooniapi_gateway/filter_plugins/ooniapi_routes.py @@ -17,11 +17,17 @@ def _locations(routes): continue for p in r.get("paths", []): prefix = p.endswith("*") + cache = None + if p in r.get("cache_long", []): + cache = "long" + elif r.get("cache"): + cache = "short" out.append({ "route": r["name"], "service": r["service"], "match": "^~" if prefix else "=", "path": p[:-1] if prefix else p, + "cache": cache, }) return out @@ -38,6 +44,10 @@ def ooniapi_gateway_route_problems(routes): locations = _locations(routes) problems = [] seen = {} + for r in routes: + for p in r.get("cache_long", []): + if p not in r.get("paths", []): + problems.append(f"{r['name']}: cache_long {p} is not one of its paths") for loc in locations: if "*" in loc["path"] or "?" in loc["path"]: problems.append(f"{loc['route']}: {loc['path']}: only exact paths and a trailing * can be served by nginx") diff --git a/ansible/roles/ooniapi_gateway/tasks/main.yml b/ansible/roles/ooniapi_gateway/tasks/main.yml index 7fc36e8b..5e812bc1 100644 --- a/ansible/roles/ooniapi_gateway/tasks/main.yml +++ b/ansible/roles/ooniapi_gateway/tasks/main.yml @@ -220,6 +220,15 @@ - name: Derive and check the gateway's locations ansible.builtin.import_tasks: routes.yml +- name: Ensure the gateway's cache directories exist + ansible.builtin.file: + path: "{{ ooniapi_gateway_cache_dir }}/{{ item }}" + state: directory + owner: "{{ ooniapi_gateway_nginx_user }}" + group: "{{ ooniapi_gateway_nginx_user }}" + mode: "0700" + loop: "{{ ooniapi_gateway_cache_zones.keys() | list }}" + - name: Render the gateway vhost ansible.builtin.template: src: gateway.conf.j2 diff --git a/ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 b/ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 index 76782f9e..dca63a30 100644 --- a/ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 +++ b/ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 @@ -24,9 +24,15 @@ map $remote_addr $ooniapi_gateway_remote_addr_anon { default 0.0.0.0; } -log_format ooniapi_gateway_fmt '$ooniapi_gateway_remote_addr_anon [$time_local] ' +log_format ooniapi_gateway_fmt '$ooniapi_gateway_remote_addr_anon $upstream_cache_status [$time_local] ' '"$request" $status $body_bytes_sent rt:$request_time "$http_referer" "$http_user_agent"'; +# response caches, see ooniapi_gateway_cache_zones in defaults/main.yml +{% for name, zone in ooniapi_gateway_cache_zones.items() %} +proxy_cache_path {{ ooniapi_gateway_cache_dir }}/{{ name }} levels=1:2 keys_zone=ooniapi_{{ name }}:{{ zone.keys_zone }} + max_size={{ zone.max_size }} inactive={{ zone.inactive }} use_temp_path=off; +{% endfor %} + server { listen 443 ssl http2; listen [::]:443 ssl http2; @@ -64,11 +70,28 @@ server { proxy_set_header X-Forwarded-Proto $scheme; proxy_read_timeout 900; + # caching, for the locations of routes marked cache/cache_long in + # routes.yaml (proxy_cache is off elsewhere). Responses are cached as long + # as their Cache-Control allows (nginx skips no-cache, private, max-age=0 + # and those setting cookies), or ooniapi_gateway_cache_default_valid + # without one. Requests with credentials never use the cache. + proxy_cache_key "$scheme$host$request_uri"; + proxy_cache_methods GET HEAD; + proxy_cache_valid 200 301 302 {{ ooniapi_gateway_cache_default_valid }}; + proxy_cache_valid any 0; + proxy_cache_lock on; + proxy_cache_lock_timeout 30s; + proxy_cache_lock_age 30s; + proxy_cache_use_stale error timeout invalid_header updating; + proxy_cache_bypass $http_authorization; + proxy_no_cache $http_authorization; + add_header X-Cache-Status $upstream_cache_status always; + {% set remote_upstreams = {'testlists': 'https://' ~ ooniapi_gateway_testlists_upstream} if ooniapi_gateway_testlists_upstream else {} %} ## --- generated from routes.yaml, see tasks/main.yml --- {% for l in ooniapi_gateway_locations %} {% if l.service in service_names %} - location {{ l.match }} {{ l.path }} { proxy_pass http://{{ l.service }}; } # {{ l.route }} + location {{ l.match }} {{ l.path }} { proxy_pass http://{{ l.service }};{% if l.cache %} proxy_cache ooniapi_{{ l.cache }};{% endif %} } # {{ l.route }} {% elif l.service in remote_upstreams %} # {{ l.service }} is not yet migrated off AWS, proxied cross-cloud until it is location {{ l.match }} {{ l.path }} { proxy_pass {{ remote_upstreams[l.service] }}; } # {{ l.route }} diff --git a/tf/modules/ooniapi_frontend/routes.yaml b/tf/modules/ooniapi_frontend/routes.yaml index 3a0bb8fe..ea1e430c 100644 --- a/tf/modules/ooniapi_frontend/routes.yaml +++ b/tf/modules/ooniapi_frontend/routes.yaml @@ -19,6 +19,10 @@ # (ALB only: they only choose between ooniprobe and its legacy) # legacy ALB only: ooniprobe_legacy is being retired and does not run # on the gateway, which sends these paths to ooniprobe +# cache gateway only: cache GET/HEAD responses of these paths in the +# short lived zone (see roles/ooniapi_gateway defaults) +# cache_long gateway only: those of its paths to cache in the long lived zone +# instead, for responses of immutable measurements - name: ooniauth_rule priority: 108 @@ -40,6 +44,7 @@ service: oonirun paths: - "/api/v2/oonirun/*" + cache: true - name: oonirun_rule_host priority: 111 @@ -138,6 +143,10 @@ - "/api/v1/measurement_meta" - "/api/v1/measurements" - "/api/v1/torsf_stats" + cache: true + cache_long: + - "/api/v1/raw_measurement" + - "/api/v1/measurement_meta" - name: oonimeasurements_rule_2 priority: 142 @@ -147,12 +156,14 @@ - "/api/v1/aggregation/*" - "/api/v1/observations" - "/api/v1/analysis" + cache: true - name: oonimeasurements_rule_3 priority: 143 service: oonimeasurements paths: - "/api/v1/detector/changepoints" + cache: true - name: testlists_rule priority: 144 From 525c6575ce10451771ac252566d6fa9633099861 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Sat, 3 Oct 2026 01:25:10 +0000 Subject: [PATCH 14/24] Invalidate a service's gateway cache when it is deployed A deploy can change what a service returns, but the gateway would keep serving responses cached from the old slot, for up to a day for measurement bodies. nginx without the commercial purge API can't drop entries, so each service's cache key now starts with a version: the image tag, which deploy.py writes into -upstream.conf as $ooniapi_cache_version_, the same file and the same reload that put the new slot in rotation. From then on the old entries are never looked up and age out of the zones; other services' caches are left alone. (The slot letter wouldn't do: slots alternate, so it would bring back responses from two deploys ago.) When nginx -t rejects the new upstream conf, deploy.py now puts the previous file back as it was, cache version included, rather than rendering it again. The image tag is checked to be a plain tag, since it ends up in an nginx string. The placeholder upstream conf the role seeds before a first deploy defines an empty version. Checked with nginx 1.27: after a deploy of oonimeasurements its cached list missed (new upstream response) then hit again, while oonirun's cached response kept hitting; redeploying the previous tag found that tag's entries again, as long as they were still valid. --- ansible/roles/ooniapi_gateway/tasks/main.yml | 3 ++ .../ooniapi_gateway/templates/gateway.conf.j2 | 8 +++-- .../ooniapi_service_deployer/files/deploy.py | 32 +++++++++++++------ .../templates/nginx_upstream.conf.tftpl | 8 +++++ 4 files changed, 38 insertions(+), 13 deletions(-) diff --git a/ansible/roles/ooniapi_gateway/tasks/main.yml b/ansible/roles/ooniapi_gateway/tasks/main.yml index 5e812bc1..8b1594c4 100644 --- a/ansible/roles/ooniapi_gateway/tasks/main.yml +++ b/ansible/roles/ooniapi_gateway/tasks/main.yml @@ -67,6 +67,9 @@ ansible.builtin.copy: dest: "/etc/nginx/conf.d/{{ item.item.name }}-upstream.conf" content: | + map "" $ooniapi_cache_version_{{ item.item.name }} { + default ""; + } upstream {{ item.item.name }} { server 127.0.0.1:{{ item.item.host_port_a }}; server 127.0.0.1:{{ item.item.host_port_b }} down; diff --git a/ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 b/ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 index dca63a30..a6563a5b 100644 --- a/ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 +++ b/ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 @@ -74,8 +74,10 @@ server { # routes.yaml (proxy_cache is off elsewhere). Responses are cached as long # as their Cache-Control allows (nginx skips no-cache, private, max-age=0 # and those setting cookies), or ooniapi_gateway_cache_default_valid - # without one. Requests with credentials never use the cache. - proxy_cache_key "$scheme$host$request_uri"; + # without one. Requests with credentials never use the cache. Each cached + # location keys on its service's $ooniapi_cache_version_, the + # image tag its blue/green deploy writes to -upstream.conf, so a + # deploy invalidates that service's responses, and only those. proxy_cache_methods GET HEAD; proxy_cache_valid 200 301 302 {{ ooniapi_gateway_cache_default_valid }}; proxy_cache_valid any 0; @@ -91,7 +93,7 @@ server { ## --- generated from routes.yaml, see tasks/main.yml --- {% for l in ooniapi_gateway_locations %} {% if l.service in service_names %} - location {{ l.match }} {{ l.path }} { proxy_pass http://{{ l.service }};{% if l.cache %} proxy_cache ooniapi_{{ l.cache }};{% endif %} } # {{ l.route }} + location {{ l.match }} {{ l.path }} { proxy_pass http://{{ l.service }};{% if l.cache %} proxy_cache ooniapi_{{ l.cache }}; proxy_cache_key "$ooniapi_cache_version_{{ l.service }}$scheme$host$request_uri";{% endif %} } # {{ l.route }} {% elif l.service in remote_upstreams %} # {{ l.service }} is not yet migrated off AWS, proxied cross-cloud until it is location {{ l.match }} {{ l.path }} { proxy_pass {{ remote_upstreams[l.service] }}; } # {{ l.route }} diff --git a/tf/modules/ooniapi_service_deployer/files/deploy.py b/tf/modules/ooniapi_service_deployer/files/deploy.py index 13892537..16eae8f1 100644 --- a/tf/modules/ooniapi_service_deployer/files/deploy.py +++ b/tf/modules/ooniapi_service_deployer/files/deploy.py @@ -20,6 +20,7 @@ import json import logging import os +import re import subprocess import sys import tempfile @@ -165,23 +166,31 @@ def deploy_host(host, ctx): log.info(f"{service} on {host}: slot {target_slot} healthy after {time.monotonic() - started:.0f}s") upstream_file = f"{service}-upstream.conf" - upstream_template = s3_fetch(ctx["bucket"], f"{service}/{upstream_file}") - - def install_upstream(live_slot): - state_a, state_b = ("", "down") if live_slot == "a" else ("down", "") - content = upstream_template.replace("__STATE_A__", state_a).replace("__STATE_B__", state_b) + upstream_path = f"/etc/nginx/conf.d/{upstream_file}" + state_a, state_b = ("", "down") if target_slot == "a" else ("down", "") + # the image tag doubles as this service's cache version on the gateway + # (see nginx_upstream.conf.tftpl): new tag, new cache keys + upstream_content = ( + s3_fetch(ctx["bucket"], f"{service}/{upstream_file}") + .replace("__STATE_A__", state_a).replace("__STATE_B__", state_b) + .replace("__CACHE_VERSION__", ctx["image_tag"]) + ) + # kept as is, to put back if nginx rejects the new one + previous_content = ssh_output(user, host, f"cat {upstream_path}") + + def install_upstream(content): scp(user, host, write_tmp(upstream_file, content), f"{STAGING_DIR}/{upstream_file}") - ssh(user, host, f"sudo mv {STAGING_DIR}/{upstream_file} /etc/nginx/conf.d/{upstream_file}") + ssh(user, host, f"sudo mv {STAGING_DIR}/{upstream_file} {upstream_path}") - install_upstream(target_slot) + install_upstream(upstream_content) if not ssh_succeeds(user, host, "sudo nginx -t"): # nginx is still running its old config, but a config that fails # nginx -t must not stay in conf.d: the next reload (another # service's deploy, a certificate renewal) would fail, and a restart - # would take every service on this host down. Put back the upstream - # of the slot still serving, which nginx -t accepted before. + # would take every service on this host down. Put back the previous + # file, slot and cache version, which nginx -t accepted before. log.error(f"{service} on {host}: nginx -t failed with slot {target_slot}'s upstream, restoring slot {active_slot}'s") - install_upstream(active_slot) + install_upstream(previous_content) if not ssh_succeeds(user, host, "sudo nginx -t"): sys.exit(f"{service} on {host}: nginx -t still fails after restoring slot {active_slot}'s upstream:" " the nginx config is broken independently of this deploy, fix it before reloading nginx") @@ -197,6 +206,9 @@ def main(): logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s", stream=sys.stdout) with open("imagedefinitions.json") as f: image_tag = json.load(f)[0]["imageUri"].rsplit(":", 1)[-1] + # it ends up in an nginx string and in compose files + if not re.fullmatch(r"[A-Za-z0-9_.-]{1,128}", image_tag): + sys.exit(f"unexpected image tag: {image_tag!r}") service = require_env("SERVICE_NAME") log.info(f"Deploying {service} image tag {image_tag}") diff --git a/tf/modules/ooniapi_service_deployer/templates/nginx_upstream.conf.tftpl b/tf/modules/ooniapi_service_deployer/templates/nginx_upstream.conf.tftpl index d239d393..640eff7f 100644 --- a/tf/modules/ooniapi_service_deployer/templates/nginx_upstream.conf.tftpl +++ b/tf/modules/ooniapi_service_deployer/templates/nginx_upstream.conf.tftpl @@ -1,3 +1,11 @@ +# Rendered by deploy.py on every deploy. __CACHE_VERSION__ becomes the +# deployed image tag: it is part of the gateway's cache key for this +# service, so the reload that puts the new slot in rotation also stops +# serving responses cached from the old one. +map "" $ooniapi_cache_version_${service_name} { + default "__CACHE_VERSION__"; +} + upstream ${service_name} { server 127.0.0.1:${host_port_a} __STATE_A__; server 127.0.0.1:${host_port_b} __STATE_B__; From d20cc6f884df102841e6df37dd7f6294934fd5d3 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Sat, 3 Oct 2026 10:28:41 +0000 Subject: [PATCH 15/24] ooniapi_service_deployer: add deploy_mode "both" A service had to choose between ECS and the Hetzner hosts. Switching one to "blue_green" removes the ECS deploy action but leaves the ECS service running, so the ALB keeps sending api.ooni.org and .prod.ooni.io traffic to tasks that no longer get new versions, until DNS moves to the hosts. "both" deploys the same build artifact to ECS and then, in the same Deploy stage, runs the blue/green deploy to the hosts. The ECS deploy runs first, so the hosts never get a version ECS rejected. Either failing fails the pipeline. The migration can then keep the single DNS cutover of the original design while every service runs in "both": the hosts are proven before the move, and DNS can go back to the ALB, which still serves the current version, if they aren't. "ecs" and "blue_green" plan exactly as before. Planned with fake inputs, one service per mode, and the account id lookup replaced by a constant: the 6 and 13 resources are identical. A "both" service gets the 13 blue/green resources and a Deploy stage with ECS "Deploy" (run order 1) then CodeBuild "DeployBlueGreen" (run order 2). Both actions need their own name and namespace. The pipeline role already allows ecs:* and codebuild:StartBuild on any resource. The per-service comments in dev and prod now mention "both", and no longer say the blue/green deploy uses systemd (it uses Docker Compose since e11b071). --- tf/environments/dev/main.tf | 30 +++++++++------- tf/environments/prod/main.tf | 30 +++++++++------- tf/modules/ooniapi_service_deployer/main.tf | 29 ++++++++------- .../ooniapi_service_deployer/variables.tf | 36 ++++++++++--------- 4 files changed, 73 insertions(+), 52 deletions(-) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 8ded4312..f8a2fab6 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -653,8 +653,9 @@ EOF module "ooniapi_ooniprobe_deployer" { source = "../../modules/ooniapi_service_deployer" - # Flip to "blue_green" to switch this service to the Docker + systemd - # blue/green deploy on the dedicated Hetzner hosts. + # "blue_green" deploys this service with Docker Compose blue/green to the + # dedicated Hetzner hosts instead of ECS; "both" deploys to ECS, then to + # the hosts, so the ALB stays current until DNS moves. deploy_mode = "ecs" service_name = "ooniprobe" @@ -803,8 +804,9 @@ module "ooniapi_ooniprobe_legacy" { module "ooniapi_reverseproxy_deployer" { source = "../../modules/ooniapi_service_deployer" - # Flip to "blue_green" to switch this service to the Docker + systemd - # blue/green deploy on the dedicated Hetzner hosts. + # "blue_green" deploys this service with Docker Compose blue/green to the + # dedicated Hetzner hosts instead of ECS; "both" deploys to ECS, then to + # the hosts, so the ALB stays current until DNS moves. deploy_mode = "ecs" service_name = "reverseproxy" @@ -1122,8 +1124,9 @@ module "fastpath_builder" { module "ooniapi_oonirun_deployer" { source = "../../modules/ooniapi_service_deployer" - # Flip to "blue_green" to switch this service to the Docker + systemd - # blue/green deploy on the dedicated Hetzner hosts. + # "blue_green" deploys this service with Docker Compose blue/green to the + # dedicated Hetzner hosts instead of ECS; "both" deploys to ECS, then to + # the hosts, so the ALB stays current until DNS moves. deploy_mode = "ecs" service_name = "oonirun" @@ -1183,8 +1186,9 @@ module "ooniapi_oonirun" { module "ooniapi_oonifindings_deployer" { source = "../../modules/ooniapi_service_deployer" - # Flip to "blue_green" to switch this service to the Docker + systemd - # blue/green deploy on the dedicated Hetzner hosts. + # "blue_green" deploys this service with Docker Compose blue/green to the + # dedicated Hetzner hosts instead of ECS; "both" deploys to ECS, then to + # the hosts, so the ALB stays current until DNS moves. deploy_mode = "ecs" service_name = "oonifindings" @@ -1243,8 +1247,9 @@ module "ooniapi_oonifindings" { module "ooniapi_ooniauth_deployer" { source = "../../modules/ooniapi_service_deployer" - # Flip to "blue_green" to switch this service to the Docker + systemd - # blue/green deploy on the dedicated Hetzner hosts. + # "blue_green" deploys this service with Docker Compose blue/green to the + # dedicated Hetzner hosts instead of ECS; "both" deploys to ECS, then to + # the hosts, so the ALB stays current until DNS moves. deploy_mode = "ecs" service_name = "ooniauth" @@ -1337,8 +1342,9 @@ module "ooniapi_ooniauth" { module "ooniapi_oonimeasurements_deployer" { source = "../../modules/ooniapi_service_deployer" - # Flip to "blue_green" to switch this service to the Docker + systemd - # blue/green deploy on the dedicated Hetzner hosts. + # "blue_green" deploys this service with Docker Compose blue/green to the + # dedicated Hetzner hosts instead of ECS; "both" deploys to ECS, then to + # the hosts, so the ALB stays current until DNS moves. deploy_mode = "ecs" service_name = "oonimeasurements" diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf index e0c4d2e1..a1e6b79c 100644 --- a/tf/environments/prod/main.tf +++ b/tf/environments/prod/main.tf @@ -542,8 +542,9 @@ resource "aws_route53_record" "ooni_wc_th" { module "ooniapi_reverseproxy_deployer" { source = "../../modules/ooniapi_service_deployer" - # Flip to "blue_green" to switch this service to the Docker + systemd - # blue/green deploy on the dedicated Hetzner hosts. + # "blue_green" deploys this service with Docker Compose blue/green to the + # dedicated Hetzner hosts instead of ECS; "both" deploys to ECS, then to + # the hosts, so the ALB stays current until DNS moves. deploy_mode = "ecs" service_name = "reverseproxy" @@ -978,8 +979,9 @@ EOF module "ooniapi_ooniprobe_deployer" { source = "../../modules/ooniapi_service_deployer" - # Flip to "blue_green" to switch this service to the Docker + systemd - # blue/green deploy on the dedicated Hetzner hosts. + # "blue_green" deploys this service with Docker Compose blue/green to the + # dedicated Hetzner hosts instead of ECS; "both" deploys to ECS, then to + # the hosts, so the ALB stays current until DNS moves. deploy_mode = "ecs" service_name = "ooniprobe" @@ -1346,8 +1348,9 @@ resource "aws_iam_role_policy" "reuploader_role" { module "ooniapi_oonirun_deployer" { source = "../../modules/ooniapi_service_deployer" - # Flip to "blue_green" to switch this service to the Docker + systemd - # blue/green deploy on the dedicated Hetzner hosts. + # "blue_green" deploys this service with Docker Compose blue/green to the + # dedicated Hetzner hosts instead of ECS; "both" deploys to ECS, then to + # the hosts, so the ALB stays current until DNS moves. deploy_mode = "ecs" service_name = "oonirun" @@ -1408,8 +1411,9 @@ module "ooniapi_oonirun" { module "ooniapi_oonifindings_deployer" { source = "../../modules/ooniapi_service_deployer" - # Flip to "blue_green" to switch this service to the Docker + systemd - # blue/green deploy on the dedicated Hetzner hosts. + # "blue_green" deploys this service with Docker Compose blue/green to the + # dedicated Hetzner hosts instead of ECS; "both" deploys to ECS, then to + # the hosts, so the ALB stays current until DNS moves. deploy_mode = "ecs" service_name = "oonifindings" @@ -1470,8 +1474,9 @@ module "ooniapi_oonifindings" { module "ooniapi_ooniauth_deployer" { source = "../../modules/ooniapi_service_deployer" - # Flip to "blue_green" to switch this service to the Docker + systemd - # blue/green deploy on the dedicated Hetzner hosts. + # "blue_green" deploys this service with Docker Compose blue/green to the + # dedicated Hetzner hosts instead of ECS; "both" deploys to ECS, then to + # the hosts, so the ALB stays current until DNS moves. deploy_mode = "ecs" service_name = "ooniauth" @@ -1566,8 +1571,9 @@ module "ooniapi_ooniauth" { module "ooniapi_oonimeasurements_deployer" { source = "../../modules/ooniapi_service_deployer" - # Flip to "blue_green" to switch this service to the Docker + systemd - # blue/green deploy on the dedicated Hetzner hosts. + # "blue_green" deploys this service with Docker Compose blue/green to the + # dedicated Hetzner hosts instead of ECS; "both" deploys to ECS, then to + # the hosts, so the ALB stays current until DNS moves. deploy_mode = "ecs" service_name = "oonimeasurements" diff --git a/tf/modules/ooniapi_service_deployer/main.tf b/tf/modules/ooniapi_service_deployer/main.tf index 5fbc02b3..dc6f2165 100755 --- a/tf/modules/ooniapi_service_deployer/main.tf +++ b/tf/modules/ooniapi_service_deployer/main.tf @@ -5,6 +5,9 @@ data "aws_caller_identity" "current" {} locals { account_id = data.aws_caller_identity.current.account_id env_label = var.environment == "prod" ? "latest" : "dev" + + deploy_ecs = contains(["ecs", "both"], var.deploy_mode) + deploy_blue_green = contains(["blue_green", "both"], var.deploy_mode) } resource "aws_iam_policy" "codebuild" { @@ -161,10 +164,10 @@ resource "aws_codebuild_project" "ooniapi" { } } -## Docker Compose blue/green deploy (deploy_mode = "blue_green") +## Docker Compose blue/green deploy (deploy_mode = "blue_green" or "both") resource "aws_s3_object" "compose_file" { - for_each = var.deploy_mode == "blue_green" ? { a = var.host_port_a, b = var.host_port_b } : {} + for_each = local.deploy_blue_green ? { a = var.host_port_a, b = var.host_port_b } : {} bucket = var.deploy_bucket key = "${var.service_name}/${var.service_name}-${each.key}.yaml" @@ -182,7 +185,7 @@ resource "aws_s3_object" "compose_file" { } resource "aws_s3_object" "nginx_upstream" { - count = var.deploy_mode == "blue_green" ? 1 : 0 + count = local.deploy_blue_green ? 1 : 0 bucket = var.deploy_bucket key = "${var.service_name}/${var.service_name}-upstream.conf" @@ -196,7 +199,7 @@ resource "aws_s3_object" "nginx_upstream" { } resource "aws_s3_object" "deploy_script" { - count = var.deploy_mode == "blue_green" ? 1 : 0 + count = local.deploy_blue_green ? 1 : 0 bucket = var.deploy_bucket key = "${var.service_name}/deploy.py" @@ -206,7 +209,7 @@ resource "aws_s3_object" "deploy_script" { } resource "aws_iam_policy" "deploy" { - count = var.deploy_mode == "blue_green" ? 1 : 0 + count = local.deploy_blue_green ? 1 : 0 description = "Policy used in trust relationship with the blue/green deploy CodeBuild project" name = "codebuild-deploy-${var.service_name}-${var.aws_region}" @@ -260,7 +263,7 @@ resource "aws_iam_policy" "deploy" { } resource "aws_iam_role" "deploy" { - count = var.deploy_mode == "blue_green" ? 1 : 0 + count = local.deploy_blue_green ? 1 : 0 assume_role_policy = < Date: Sun, 4 Oct 2026 05:18:21 +0000 Subject: [PATCH 16/24] prod: leave the blue/green deploy secrets out until a service moves Applying prod as it was would create 7 Secrets Manager secrets: a JSON copy of each service's credentials (Postgres URL, JWT key, ClickHouse URL, ooniauth's AWS keys) and the deploy SSH key container. Every prod service is still deploy_mode = "ecs", so nothing would read them. They are commented out in place, with each deployer's pre-wired env_vars/secrets block, so moving a prod service to "blue_green" or "both" starts by uncommenting both. Against main, tf/environments/prod now differs only by comments and the explicit deploy_mode = "ecs" (the default), so the prod plan of this branch is the ooniapi_frontend listener rules moving to their routes.yaml addresses: planned against a state built from main's module with prod's inputs, 18 moved, 0 to add, change or destroy. terraform validate and fmt pass. --- tf/environments/prod/main.tf | 299 ++++++++++++++++++----------------- 1 file changed, 155 insertions(+), 144 deletions(-) diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf index a1e6b79c..5a544366 100644 --- a/tf/environments/prod/main.tf +++ b/tf/environments/prod/main.tf @@ -287,6 +287,11 @@ resource "aws_secretsmanager_secret_version" "oonipg_url" { ) } +# Left out of prod until a prod service moves to deploy_mode "blue_green" +# or "both": applying them would copy prod's credentials into new Secrets +# Manager secrets that nothing reads yet. Uncomment together with the +# matching block in that service's deployer below. +# ### Blue/green deploy secrets (Docker + systemd on dedicated Hetzner hosts) # # These mirror the values already passed as `task_secrets` to the ECS task @@ -296,86 +301,86 @@ resource "aws_secretsmanager_secret_version" "oonipg_url" { # slot's env file. Values are pulled from the same underlying data sources # used by the ECS `task_secrets` maps, so both deploy paths stay in sync # automatically. - -data "aws_secretsmanager_secret_version" "ooniapi_user_access_key_id" { - secret_id = module.ooniapi_user.aws_access_key_id_arn -} - -data "aws_secretsmanager_secret_version" "ooniapi_user_secret_access_key" { - secret_id = module.ooniapi_user.aws_secret_access_key_arn -} - -locals { - ooniapi_deploy_service_secrets = { - reverseproxy = { - PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value - } - ooniprobe = { - POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value - JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value - PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value - CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_write_url.value - ANONC_SECRET_KEY = data.aws_ssm_parameter.anonc_secret_key.value - } - oonirun = { - POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value - JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value - PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value - CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_oonirun_url.value - } - oonifindings = { - POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value - JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value - PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value - } - ooniauth = { - POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value - JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value - PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value - ACCOUNT_ID_HASHING_KEY = data.aws_ssm_parameter.account_id_hashing_key.value - AWS_SECRET_ACCESS_KEY = data.aws_secretsmanager_secret_version.ooniapi_user_secret_access_key.secret_string - AWS_ACCESS_KEY_ID = data.aws_secretsmanager_secret_version.ooniapi_user_access_key_id.secret_string - } - oonimeasurements = { - POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value - JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value - PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value - CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_oonimeasurements_url.value - ACCOUNT_ID_HASHING_KEY = data.aws_ssm_parameter.account_id_hashing_key.value - } - } -} - -resource "aws_secretsmanager_secret" "ooniapi_deploy_service_secrets" { - for_each = local.ooniapi_deploy_service_secrets - - name = "oonidevops/ooniapi/${each.key}/service_secrets" - tags = local.tags -} - -resource "aws_secretsmanager_secret_version" "ooniapi_deploy_service_secrets" { - for_each = local.ooniapi_deploy_service_secrets - - secret_id = aws_secretsmanager_secret.ooniapi_deploy_service_secrets[each.key].id - secret_string = jsonencode(each.value) -} - +# +# data "aws_secretsmanager_secret_version" "ooniapi_user_access_key_id" { +# secret_id = module.ooniapi_user.aws_access_key_id_arn +# } +# +# data "aws_secretsmanager_secret_version" "ooniapi_user_secret_access_key" { +# secret_id = module.ooniapi_user.aws_secret_access_key_arn +# } +# +# locals { +# ooniapi_deploy_service_secrets = { +# reverseproxy = { +# PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value +# } +# ooniprobe = { +# POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value +# JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value +# PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value +# CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_write_url.value +# ANONC_SECRET_KEY = data.aws_ssm_parameter.anonc_secret_key.value +# } +# oonirun = { +# POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value +# JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value +# PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value +# CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_oonirun_url.value +# } +# oonifindings = { +# POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value +# JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value +# PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value +# } +# ooniauth = { +# POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value +# JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value +# PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value +# ACCOUNT_ID_HASHING_KEY = data.aws_ssm_parameter.account_id_hashing_key.value +# AWS_SECRET_ACCESS_KEY = data.aws_secretsmanager_secret_version.ooniapi_user_secret_access_key.secret_string +# AWS_ACCESS_KEY_ID = data.aws_secretsmanager_secret_version.ooniapi_user_access_key_id.secret_string +# } +# oonimeasurements = { +# POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value +# JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value +# PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value +# CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_oonimeasurements_url.value +# ACCOUNT_ID_HASHING_KEY = data.aws_ssm_parameter.account_id_hashing_key.value +# } +# } +# } +# +# resource "aws_secretsmanager_secret" "ooniapi_deploy_service_secrets" { +# for_each = local.ooniapi_deploy_service_secrets +# +# name = "oonidevops/ooniapi/${each.key}/service_secrets" +# tags = local.tags +# } +# +# resource "aws_secretsmanager_secret_version" "ooniapi_deploy_service_secrets" { +# for_each = local.ooniapi_deploy_service_secrets +# +# secret_id = aws_secretsmanager_secret.ooniapi_deploy_service_secrets[each.key].id +# secret_string = jsonencode(each.value) +# } +# # Shared by every service's blue/green deploy job. The private key itself is # generated and rotated out-of-band (see the deploy README); Terraform only # owns the secret container, not its value. -resource "aws_secretsmanager_secret" "ooniapi_deploy_ssh_key" { - name = "oonidevops/ooniapi/deploy_ssh_key" - tags = local.tags -} - -resource "aws_secretsmanager_secret_version" "ooniapi_deploy_ssh_key" { - secret_id = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.id - secret_string = "REPLACE_ME: populate out-of-band with the \"deploy\" user's SSH private key" - - lifecycle { - ignore_changes = [secret_string] - } -} +# resource "aws_secretsmanager_secret" "ooniapi_deploy_ssh_key" { +# name = "oonidevops/ooniapi/deploy_ssh_key" +# tags = local.tags +# } + +# resource "aws_secretsmanager_secret_version" "ooniapi_deploy_ssh_key" { +# secret_id = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.id +# secret_string = "REPLACE_ME: populate out-of-band with the \"deploy\" user's SSH private key" +# +# lifecycle { +# ignore_changes = [secret_string] +# } +# } module "geoip_bucket" { source = "../../modules/s3_bucket" @@ -560,13 +565,14 @@ module "ooniapi_reverseproxy_deployer" { ecs_service_name = module.ooniapi_reverseproxy.ecs_service_name ecs_cluster_name = module.ooniapi_cluster.cluster_name - # Pre-wired for the future flip to deploy_mode = "blue_green" - env_vars = { - TARGET_URL = "https://backend-fsn.ooni.org/" - } - secrets = keys(local.ooniapi_deploy_service_secrets.reverseproxy) - service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["reverseproxy"].arn - deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn + # Pre-wired for the future flip to deploy_mode = "blue_green" or "both"; + # commented out with the blue/green deploy secrets above + # env_vars = { + # TARGET_URL = "https://backend-fsn.ooni.org/" + # } + # secrets = keys(local.ooniapi_deploy_service_secrets.reverseproxy) + # service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["reverseproxy"].arn + # deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn } module "ooniapi_reverseproxy" { @@ -997,22 +1003,23 @@ module "ooniapi_ooniprobe_deployer" { ecs_service_name = module.ooniapi_ooniprobe.ecs_service_name ecs_cluster_name = module.ooniapi_cluster.cluster_name - # Pre-wired for the future flip to deploy_mode = "blue_green" - env_vars = { - # hardcoded IP for fastpath2.prod.prod.ooni.io - FASTPATH_URL = "http://10.0.0.32:8472" - FASTPATH_URLS = jsonencode([for h in local.fastpath_hosts : "http://${h}:8472"]) - FAILED_REPORTS_BUCKET = aws_s3_bucket.ooniprobe_failed_reports.bucket - COLLECTOR_ID = 4 # be sure this is different from dev - CONFIG_BUCKET = aws_s3_bucket.ooni_private_config_bucket.bucket - TOR_TARGETS = "tor_targets.json" - PSIPHON_CONFIG = "psiphon_config.json" - ANONC_MANIFEST_BUCKET = aws_s3_bucket.anoncred_manifests.bucket - ANONC_MANIFEST_FILE = "manifest.json" - } - secrets = keys(local.ooniapi_deploy_service_secrets.ooniprobe) - service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["ooniprobe"].arn - deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn + # Pre-wired for the future flip to deploy_mode = "blue_green" or "both"; + # commented out with the blue/green deploy secrets above + # env_vars = { + # # hardcoded IP for fastpath2.prod.prod.ooni.io + # FASTPATH_URL = "http://10.0.0.32:8472" + # FASTPATH_URLS = jsonencode([for h in local.fastpath_hosts : "http://${h}:8472"]) + # FAILED_REPORTS_BUCKET = aws_s3_bucket.ooniprobe_failed_reports.bucket + # COLLECTOR_ID = 4 # be sure this is different from dev + # CONFIG_BUCKET = aws_s3_bucket.ooni_private_config_bucket.bucket + # TOR_TARGETS = "tor_targets.json" + # PSIPHON_CONFIG = "psiphon_config.json" + # ANONC_MANIFEST_BUCKET = aws_s3_bucket.anoncred_manifests.bucket + # ANONC_MANIFEST_FILE = "manifest.json" + # } + # secrets = keys(local.ooniapi_deploy_service_secrets.ooniprobe) + # service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["ooniprobe"].arn + # deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn } module "ooniapi_ooniprobe" { @@ -1366,11 +1373,12 @@ module "ooniapi_oonirun_deployer" { ecs_service_name = module.ooniapi_oonirun.ecs_service_name ecs_cluster_name = module.ooniapi_cluster.cluster_name - # Pre-wired for the future flip to deploy_mode = "blue_green" - env_vars = {} - secrets = keys(local.ooniapi_deploy_service_secrets.oonirun) - service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["oonirun"].arn - deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn + # Pre-wired for the future flip to deploy_mode = "blue_green" or "both"; + # commented out with the blue/green deploy secrets above + # env_vars = {} + # secrets = keys(local.ooniapi_deploy_service_secrets.oonirun) + # service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["oonirun"].arn + # deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn } module "ooniapi_oonirun" { @@ -1429,11 +1437,12 @@ module "ooniapi_oonifindings_deployer" { ecs_service_name = module.ooniapi_oonifindings.ecs_service_name ecs_cluster_name = module.ooniapi_cluster.cluster_name - # Pre-wired for the future flip to deploy_mode = "blue_green" - env_vars = {} - secrets = keys(local.ooniapi_deploy_service_secrets.oonifindings) - service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["oonifindings"].arn - deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn + # Pre-wired for the future flip to deploy_mode = "blue_green" or "both"; + # commented out with the blue/green deploy secrets above + # env_vars = {} + # secrets = keys(local.ooniapi_deploy_service_secrets.oonifindings) + # service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["oonifindings"].arn + # deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn } module "ooniapi_oonifindings" { @@ -1492,26 +1501,27 @@ module "ooniapi_ooniauth_deployer" { ecs_service_name = module.ooniapi_ooniauth.ecs_service_name ecs_cluster_name = module.ooniapi_cluster.cluster_name - # Pre-wired for the future flip to deploy_mode = "blue_green" - env_vars = { - AWS_REGION = var.aws_region - EMAIL_SOURCE_ADDRESS = module.ooniapi_user.email_address - SESSION_EXPIRY_DAYS = 2 - LOGIN_EXPIRY_DAYS = 7 - ADMIN_EMAILS = jsonencode([ - "maja@ooni.org", - "arturo@ooni.org", - "mehul@ooni.org", - "norbel@ooni.org", - "maria@ooni.org", - "admin+dev@ooni.org", - "luis@openobservatory.org", - "contact@openobservatory.org" - ]) - } - secrets = keys(local.ooniapi_deploy_service_secrets.ooniauth) - service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["ooniauth"].arn - deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn + # Pre-wired for the future flip to deploy_mode = "blue_green" or "both"; + # commented out with the blue/green deploy secrets above + # env_vars = { + # AWS_REGION = var.aws_region + # EMAIL_SOURCE_ADDRESS = module.ooniapi_user.email_address + # SESSION_EXPIRY_DAYS = 2 + # LOGIN_EXPIRY_DAYS = 7 + # ADMIN_EMAILS = jsonencode([ + # "maja@ooni.org", + # "arturo@ooni.org", + # "mehul@ooni.org", + # "norbel@ooni.org", + # "maria@ooni.org", + # "admin+dev@ooni.org", + # "luis@openobservatory.org", + # "contact@openobservatory.org" + # ]) + # } + # secrets = keys(local.ooniapi_deploy_service_secrets.ooniauth) + # service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["ooniauth"].arn + # deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn } module "ooniapi_ooniauth" { @@ -1590,20 +1600,21 @@ module "ooniapi_oonimeasurements_deployer" { ecs_cluster_name = module.oonitier1plus_cluster.cluster_name # ecs_cluster_name = module.ooniapi_cluster.cluster_name - # Pre-wired for the future flip to deploy_mode = "blue_green" - env_vars = { - # it has to be a json-compliant array - OTHER_COLLECTORS = jsonencode([for h in local.fastpath_hosts : "http://${h}:8475"]) - BASE_URL = "https://api.ooni.io" - S3_BUCKET_NAME = "ooni-data-eu-fra" - VALKEY_URL = local.ooniapi_valkey_url - RATE_LIMITS = "4000/hour;400000/day;200000/7day" - RATE_LIMITS_WHITELISTED_IPADDRS = jsonencode(["5.9.112.244"]) - RATE_LIMITS_UNMETERED_PAGES = jsonencode(["/metrics", "/health"]) - } - secrets = keys(local.ooniapi_deploy_service_secrets.oonimeasurements) - service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["oonimeasurements"].arn - deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn + # Pre-wired for the future flip to deploy_mode = "blue_green" or "both"; + # commented out with the blue/green deploy secrets above + # env_vars = { + # # it has to be a json-compliant array + # OTHER_COLLECTORS = jsonencode([for h in local.fastpath_hosts : "http://${h}:8475"]) + # BASE_URL = "https://api.ooni.io" + # S3_BUCKET_NAME = "ooni-data-eu-fra" + # VALKEY_URL = local.ooniapi_valkey_url + # RATE_LIMITS = "4000/hour;400000/day;200000/7day" + # RATE_LIMITS_WHITELISTED_IPADDRS = jsonencode(["5.9.112.244"]) + # RATE_LIMITS_UNMETERED_PAGES = jsonencode(["/metrics", "/health"]) + # } + # secrets = keys(local.ooniapi_deploy_service_secrets.oonimeasurements) + # service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["oonimeasurements"].arn + # deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn } module "ooniapi_oonimeasurements" { From bb14823386adc09d83a399364213f7cafb3872d3 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Sun, 4 Oct 2026 05:19:35 +0000 Subject: [PATCH 17/24] ooniapi_service_deployer: make the secondary deploy host optional Dev needs one Hetzner host: an outage while it reboots costs nothing, and backend-hel is free for it. With both hosts required, a dev service would have to name backend-hel twice, and deploy.py would then deploy to it twice, the second time waiting out the drain timeout (300 s) for the slot it had just taken out of rotation. Without deploy_host_secondary the deploy job gets no DEPLOY_HOST_SECONDARY and deploy.py deploys to the primary alone. Planned with the deployer test module in each mode, a service with both hosts plans the same resources as before (only the deploy.py object changes); without the secondary, only that environment variable is left out. --- .../files/__pycache__/deploy.cpython-314.pyc | Bin 0 -> 12336 bytes .../ooniapi_service_deployer/files/deploy.py | 8 ++++++-- tf/modules/ooniapi_service_deployer/main.tf | 9 ++++++--- .../ooniapi_service_deployer/variables.tf | 2 +- 4 files changed, 13 insertions(+), 6 deletions(-) create mode 100644 tf/modules/ooniapi_service_deployer/files/__pycache__/deploy.cpython-314.pyc diff --git a/tf/modules/ooniapi_service_deployer/files/__pycache__/deploy.cpython-314.pyc b/tf/modules/ooniapi_service_deployer/files/__pycache__/deploy.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6ab989a4d2488c3630fbc8701778a64647983eb4 GIT binary patch literal 12336 zcmcIKTWlLwb~EIV9BN2Cskdc~Y`tiUlr1Z^EXxmBlC6jJ@JOa(M>a!~BZ)CznHkA8 z^C+`hY-DeCrQLcP26YiG8ekbIj{;uXsEhQY zJ@*cWqGZM1ZP6?7%*=hBdtUe4bMO8_YXJf27yt6v8^5*_#J{2sE$Oo_Pd4BsMqDL0 za)^-BCn=HmTPNx8Q!nZ9laeU>G)M;gG)hMNG)X2W!4;2_63yu(^A&Q!rVX?CY#=y& zC#il*3`bQGl7%w>v~os(g`5ds5k~_o=F9*~xB`Hs90Rb7E8#4aM1YiRoE5&yxk7*y zS1P%pu2%`d2lL!TT&ZXvIdE&2#E5B}iNJ&|_nLc|FyvlyFD^oxbcA$~E!q6p z8ieq)QL8pVYj2xRAQ<5yEJTM+>!F@-!KCbDv?b^lML?Mq_&4YM0`Ft_(1KqGhk|@a zVquzo&$R&G&Dsr$E-WghY^kE3t5+xX_iZRGxX_orZ{%)3A7=$5ZDA64E&ya2f!8rYZ40iTH=7CFD7zlmM!%!C9aD& zL~=S#&rzJdUFRZClN15OHGIPe^PvzBx0`Je6_Qm9-Z|cTeM+Gu{-y*J)d#|}3gr*Y zgca(Ius@^}i1XJXLfFfTqN0b!Kvums=u?c>Z+L`RQ6+1eV1Zc(RT?-1FJc#f81d9d z80hy0-y2-K67TzX@T0-RLqMP#e&Rj=ASO)VvOz97 zd5;E=m`a>UHYEF#hvhvT@SPe4kY-NF)TtNDY65rkr_2hm_egf$6BIPxmSDCNMpUSS zlBrKgrO@k%Nb>^k4$n)GdC9I<8CDO)icyq&kS(zAv1P&_0ODM>h0qG6Cr}N<;6G;G z^2Ahzxv<7e-)E*}YWf8P_&eMqW+uF;X(whz^qb7yNm~$ECXn83ORk)K&eNitVVb~e z$<`(f#AcfbB;VW8RibNUqfSP&n?d9Rn}eWI`*%1)z7QvhNF*dMu|`)(J21QlOM zoKHxJ8M)2@-^nO|Sl2C90=<~whv&_K&Dmna^J_sRly(xym#u`_vi_3w7cC^)k}GH1G@*{@ zrqLRP6IxJ^I$%Cf*dkIcoztM%Ls&+Nw7Ra`I(&?Dlk>WHy*uY_M1Xq}NT#MYQvEW0G(&Z7n5~PxrJ4_Fmfu`2DqdQ+z3}~;&-6rrZK-Cd=KD3@h{(iz)?IwM z-0-V%=5(`pizk7YkA)B^JqTgm@~1Hw)_MbH2%BQ1cwvZHh7u?c48*njDTd=Xv3lVwuG4CM5xUi74%Gs7s1a2Axg` zO#toc0xKX9u#>Qg;0A7w5E2pYs?I8A^~y69$Wz*u8i

8Xn+Q>tCjf=UTI>s8h%r zV|E&=9o7|f(K-VC3&`&hkkg|WZU}w}{A%x9FzmA%1hjT4CT}<-frGA4kjbl!v8#Yo z7_DMiT{GjxZYdZMMxf*>{6xgI+ookxhOR;? zRQ)w&e1GsegNsucs%o7nij6!oLGDt0R;lRH-fqKZ-OT9%vo~KIkoCi2@inY%9>P=Y zsqxgtU?0K}ZLo*YQUum8j?B*q*XA)GYxA_)=*j6^I<)FdB43p1$hWw;UDRalBv?UQ z&)A{YSi6(!qS@JFn>#ERa~HCrCJ-~UKH)UIwQm~tZT{o>*4BgwtP@;8{#;#DTb^?i zF}PrR+;tbWr63Tn%hLuYCU&3$#(=?8u(&9oh1F#MSlB^pwPCD3!Jb#>VYDe z@Rk_QpDP+AfdWRCF@K{B_!sBnKT1PRNq$bm^haY|iu@L?%w+^mj^|f_ zODj952+dXDoN5;h-VPNp=dG_KSCfxJUOm00s*wU$!A|QeR|_-PmEU@^#Wb?GI9D5# z^|ATX_g;49;=hygrZJ1v%^xyP+32-}$OFc2dBWG~;c6v1<+HftKdbUNTcWu%`d)o}; zL`tp_ky2ObRb5vpXw*koqtLbiC(MmjEOqwH-Zh`f2g}PY0@{*od5cG+bQHo`kcGhm<2BDs&)BIPf$1-Or#E!fcx9CFz-9pACM(^dY$@>jmqj#O-0I}T6Q zpc!ehAE|T^u4-3}tM(AZ9f43K9jS6v0go#-ZQ=A*x$V;0YVtb!E|!{2ZZ+jzb1!)^ za}>)=H@3gWw$L4oTlpu9%l?Vd+JnbD4%RN~WP?6KA8LB3tgCpRz@LVmQw>yg-Y*catu%F#XT9y>JvQv zkX!NxdDwFRrr{79t?6bX0>9u7&x_faurmRFBm%MHXpzWzX2Wgl4Ub`Yi- z{gTLr13nhd1SXC9Rmf`!g93o=NLY~ETRuF|l5SR=zS_tyvQb;xR?^|#q9(Q#wPzL5 zu8@wX!FkIQ3`DJ4Xeqn`5+(EiP^@nE&}i@30rynzS+`p;`-893;khNk7mCF! z@Q$z$ne&9iXs35xfWgEWC<=ORIuMW8SJ0T7VlpR55!l`z>^R~)0DlL&qo(#b-V=b( zj!k{KaYQ=DU{kkB>}VMq0)*8%w^+}N1V~2p&I2>iQn8KoTmyO|_Nsy$E!tL7A;oB+ zV?H8E5EckJVY345Fh4?YJaV%gWjKB93gdQ9PW4U=xO;&SxsTKEQQ~&@_4W-8xLpI> z{vi&BiD4r* zgi6t3u2?lnxpA;%^Am?+8lZ}ZHXF?`NNRYpI4f*!t)e^@o5u zJb;iCxW$HNpe_`u6l-H={5QcqvDbJoR6L*+2zz`8Ej*KGZQdZqvPi?URMWG+1+?MD zoZmZ#MpH4yCOW`OSkc-4fxEH6;r3K@z4nBtEpdOr#& zAWH%{0iKWG5S?NY;B?ais~CzQs8anpdfSDYLUbZ1D&}AqEC!s0dU3Q|K=Eq15zDt#Ly;r&^jG}xdZ*hZ&l|SU=-Dn+AD71Jej07iMyoA@j+y}DriY03;Rc*XfSVOrDAh-$hN} zCtATY{1==_6qYR8e-MhDU1tgx7k+RIhe+`mgtUBi`DlC`osp$%vh{F=I`Vbtu65e7 z=#wj2lBUEh`2Z)|CZFhaWyXIqGVk%`RS$n^Cm27QI`nUOGAP{SuCeQ8l__z_K5iWdLuAyc2&_mF9e^{<?~&8JptRxO{8%hRvRuX$w8+j7Y}8T#D~+Okw|r(n4gf==tzjY)l` zdjF61rpjU?>#c`UVy3lwdGB)Yi_*qb{p#q$Q(SB$d*rfgTD%pf;#2W6@}5Jfk<@Yd z)RcVTb$JT0~`BL1IVB+sU__(Yic5c0VPvS_z4guxq z zY#Yu{BaaG*%9^%qt#B6eRw^hr_h;yVY{7cr zE5E#+y0Y4^IrZlC$8ykhapMrp-|&L2C|rF#+=AC|T)n%0?;<=zjv z<1L9kx%I@u#*>+vlN*+zrNKLcantgx1cgt6TT=T|b@B~}5b*}-$ELs}T9W&db*b>Leg7ExRVaPNC0}~|;hEQEk53-pWqwAs z&St2&uRvt{@pomLGd8+j(VDDF7Rk0QcyH-Qc{43XfAsds+xID3?Cj!2I1e+IESD|6 z`jfh>OkGKi%WO}YKK?s%@$Jgxx`ZihZv92&rYFI-qn`XYI*pYZ1ZL)AJojVYFQg{ zV&$!s;OCvPb>a(3bxe?z%_VdKjC{c-tzav`OUxDQbn~VmzdYwF=9+-d_~H{rj@065 zu-8N@4NhDxjmGP%fZZl4LGaT|Mn+3Pu+U{I*r_T zoF*Vzw>`o98Lm*n3R)LU>A2#&kxH}?xYE4bvRtlhisZ`kN^)@^t}-vTDp$XHT4N89 zhZ%D8wxx;v+cMyjtI_&-{siReO|6Wp{XdPdOY7(PF}OOdjN6^37j4gD3YxkT-|DsY z+n?=bIKRn7-#Jz*cajn6e?g}Pt-VHUAG|;mmzrpQ9@iK5&o$(0rq(aS{V9m%gGGoK zrVna;UYy*Nq!G9PgJR_9eE-Ea00zi&kJMwwm)4uj9#8I#HZ`?+M2J8IdArCqwFdYF zK7bDX@mCP;s6pVwF52~q0T(Go*gyq661v`m_WjXv99r@5GyV`ngTf)v`Gyz{DKyT$ zAov9*OlmAz{#vi&vd0lU;5h1bI?>oQ4l@ zuA#mG_gL@fK(wGA-@L@JCT6o9`%O_hoO(hi1l`MYquU7RB8^hrKQM7_{E{18L>n1^ z=RhAfFy-#$#)L~yj&~;5Cs&H{YF>aLv%#Zm?&%ACBLhibYT3#lz?tPDAs$ndi@T(h=VwCg@v(l${{4SLvR5l6xN3;6rZRAS!|(0GTV#bA8ap$t z?$wYKt0&GB<`RXr!HQ83BLUcr=>%Re&pIDm6B**D1>oM6vq(pmQh6QS)9~;5EX=&r0`S7 z{SExYFM(DN&7hbys`3FcgQQ55+xyaV}BvlkvpxLtFcr?eKlu;gq=2o3@=;5gyu3FPhP5cMDvpgnOwycY5Ti z11b9&)ANAo!P^*gDb=2|e_AJ(9LdmKUl&%#o#{e*>}=MtT#UwB;?;@$Nq_P+*@`Zu ze2uWEPn4~hn=>+XaBd^*^xz*USR z>vZ8=M&9K}7dzMK;=B9f;dCjyl`UUdr_1lYl{lWRJedEs`xE^qr++s5)8U_u{d7z| zG?Z=}UTZurH=h68n`XIny7FGxJ?`VnA6@?V>PJ_T4Z^ufvW?x}Ryg-2xtj4{^tw1z0GmaCmam7$M*SzNwIZ2;c=U(mas=!uewXUBlp zz#-Y({LQmcAOKKFHaC4EBJb@pbI(wl;U7*k4ikodYGa0uoBx?$hD*!}?RFz+aJ%is zZ*-1ul%0e?hF2N{=^5c~^`YywU>c5l;38i{a)kIB!rwr5(GrmV5SW%JHL!jB-~qT5 z=NIlknVs~10up7f65xQ45dIPYEYSDR8@^G9V;lIIh9Bi2heCKAfMSM##W~P3h, never as @@ -229,7 +229,11 @@ def main(): "secrets": json.loads(secretsmanager_get(require_env("SERVICE_SECRETS_ARN"))), } - for host in (require_env("DEPLOY_HOST_PRIMARY"), require_env("DEPLOY_HOST_SECONDARY")): + # a single host is enough where downtime while it reboots is acceptable + hosts = [require_env("DEPLOY_HOST_PRIMARY")] + if os.environ.get("DEPLOY_HOST_SECONDARY"): + hosts.append(os.environ["DEPLOY_HOST_SECONDARY"]) + for host in hosts: deploy_host(host, ctx) diff --git a/tf/modules/ooniapi_service_deployer/main.tf b/tf/modules/ooniapi_service_deployer/main.tf index dc6f2165..d6f635b7 100755 --- a/tf/modules/ooniapi_service_deployer/main.tf +++ b/tf/modules/ooniapi_service_deployer/main.tf @@ -334,9 +334,12 @@ resource "aws_codebuild_project" "deploy" { name = "DEPLOY_HOST_PRIMARY" value = var.deploy_host_primary } - environment_variable { - name = "DEPLOY_HOST_SECONDARY" - value = var.deploy_host_secondary + dynamic "environment_variable" { + for_each = var.deploy_host_secondary == null ? [] : [var.deploy_host_secondary] + content { + name = "DEPLOY_HOST_SECONDARY" + value = environment_variable.value + } } environment_variable { name = "DEPLOY_SSH_USER" diff --git a/tf/modules/ooniapi_service_deployer/variables.tf b/tf/modules/ooniapi_service_deployer/variables.tf index 1414c445..59884d5d 100644 --- a/tf/modules/ooniapi_service_deployer/variables.tf +++ b/tf/modules/ooniapi_service_deployer/variables.tf @@ -145,7 +145,7 @@ variable "deploy_host_primary" { } variable "deploy_host_secondary" { - description = "Hostname/IP of the secondary dedicated host to deploy to. Required when deploy_mode is \"blue_green\" or \"both\"." + description = "Hostname/IP of the secondary dedicated host to deploy to, after the primary. Optional: without it the service runs on the primary host alone, as dev does." type = string default = null } From 2520513e227121df07c1b83e33fc7c2aeda2019c Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Sun, 4 Oct 2026 05:24:20 +0000 Subject: [PATCH 18/24] dev: run the gateway and the blue/green services on backend-hel alone backend-hel has been re-imaged and is free; one host is enough for dev. Until now nothing said where a dev service would go: the pre-wired deployers had no host, ports, network or deploy bucket, and the gateway playbook targeted backend-fsn, which still serves the legacy API, as well. - tf/environments/dev: each deployer gets backend-hel as its only host, its slot ports, the ooniapi docker network and container port 80, and a deploy bucket for the compose files, upstream confs and deploy.py. The services stay deploy_mode = "ecs", so the only resource this adds is the bucket. - host_vars/backend-hel.ooni.org/ooniapi_gateway.yml: the gateway answers for api.dev.ooni.io and .dev.ooni.io, with the same ports. testlists stays on its EC2 instance, which serves only plain HTTP, so the gateway proxies it over HTTPS through the ALB at testlists.dev.ooni.io; that name stays on the ALB at the DNS cutover. The dev names' certificate can only be issued after the cutover; the file says how. - deploy-ooniapi-gateway.yml targets backend-hel only, and installs the nginx and dehydrated roles the gateway role expects first. Rendered with these settings, the gateway vhost passes nginx -t (nginx 1.27) with all routes.yaml locations, the five direct-host vhosts and the testlists passthrough. terraform validate and fmt pass for dev. --- ansible/deploy-ooniapi-gateway.yml | 10 +++- .../backend-hel.ooni.org/ooniapi_gateway.yml | 29 +++++++++ tf/environments/dev/main.tf | 59 +++++++++++++++++++ 3 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 ansible/host_vars/backend-hel.ooni.org/ooniapi_gateway.yml diff --git a/ansible/deploy-ooniapi-gateway.yml b/ansible/deploy-ooniapi-gateway.yml index f65c03a9..1ff9871c 100644 --- a/ansible/deploy-ooniapi-gateway.yml +++ b/ansible/deploy-ooniapi-gateway.yml @@ -1,7 +1,15 @@ --- +# backend-hel runs the dev API services (host_vars/backend-hel.ooni.org/ +# ooniapi_gateway.yml). backend-fsn still serves the legacy API and gets the +# gateway only when prod moves. nginx and dehydrated (with ssl_domains from +# host_vars) come first: the gateway role expects both on the host. - name: Bootstrap the tier0 blue/green gateway on dedicated backend hosts - hosts: backend-fsn.ooni.org:backend-hel.ooni.org + hosts: backend-hel.ooni.org become: true roles: + - role: nginx + tags: nginx + - role: dehydrated + tags: dehydrated - role: ooniapi_gateway tags: ooniapi_gateway diff --git a/ansible/host_vars/backend-hel.ooni.org/ooniapi_gateway.yml b/ansible/host_vars/backend-hel.ooni.org/ooniapi_gateway.yml new file mode 100644 index 00000000..c47c31ca --- /dev/null +++ b/ansible/host_vars/backend-hel.ooni.org/ooniapi_gateway.yml @@ -0,0 +1,29 @@ +# backend-hel runs the dev API services, deployed blue/green by their +# deployers in tf/environments/dev (deploy_host_primary), behind the gateway +# vhost of roles/ooniapi_gateway. It answers for the names the dev ALB +# answers for; those keep pointing at the ALB until the DNS cutover. +ooniapi_gateway_primary_domain: api.dev.ooni.io +ooniapi_gateway_direct_domain_suffix: dev.ooni.io + +# testlists stays on its EC2 instance, which only serves plain HTTP: proxy it +# over HTTPS through the dev ALB. testlists.dev.ooni.io must therefore stay on +# the ALB at the cutover. +ooniapi_gateway_testlists_upstream: testlists.dev.ooni.io + +# Must match ooniapi_deploy_ports in tf/environments/dev/main.tf +ooniapi_gateway_services: + - {name: reverseproxy, host_port_a: 18001, host_port_b: 18002} + - {name: ooniprobe, host_port_a: 18011, host_port_b: 18012} + - {name: oonirun, host_port_a: 18021, host_port_b: 18022} + - {name: oonifindings, host_port_a: 18031, host_port_b: 18032} + - {name: ooniauth, host_port_a: 18041, host_port_b: 18042} + - {name: oonimeasurements, host_port_a: 18051, host_port_b: 18052} + +# The certificate for the dev names can only be issued once they resolve +# here. At the cutover, after moving their DNS: add this line to +# ssl_domains, run the playbook with --tags dehydrated, then set +# ooniapi_gateway_cert_ready to true and run it again. +# - "api.dev.ooni.io ooniauth.dev.ooni.io ooniprobe.dev.ooni.io oonirun.dev.ooni.io oonifindings.dev.ooni.io oonimeasurements.dev.ooni.io" +ssl_domains: + - "backend-hel.ooni.org" +ooniapi_gateway_cert_ready: false diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index f8a2fab6..8192e4f1 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -347,6 +347,29 @@ resource "aws_secretsmanager_secret_version" "ooniapi_deploy_ssh_key" { } } +# Where the blue/green deploy job puts the dev services: backend-hel alone. +# The ports must match ooniapi_gateway_services in +# ansible/host_vars/backend-hel.ooni.org/ooniapi_gateway.yml. +locals { + ooniapi_deploy_host = "backend-hel.ooni.org" + ooniapi_deploy_network = "ooniapi" + ooniapi_deploy_ports = { + reverseproxy = [18001, 18002] + ooniprobe = [18011, 18012] + oonirun = [18021, 18022] + oonifindings = [18031, 18032] + ooniauth = [18041, 18042] + oonimeasurements = [18051, 18052] + } +} + +# The compose files, nginx upstream confs and deploy.py the deploy jobs copy +# to the host +resource "aws_s3_bucket" "ooniapi_deploy" { + bucket = "ooniapi-deploy-${var.aws_region}-${random_id.artifact_id.hex}" + tags = local.tags +} + resource "random_id" "artifact_id" { byte_length = 4 } @@ -686,6 +709,12 @@ module "ooniapi_ooniprobe_deployer" { secrets = keys(local.ooniapi_deploy_service_secrets.ooniprobe) service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["ooniprobe"].arn deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn + deploy_bucket = aws_s3_bucket.ooniapi_deploy.bucket + deploy_host_primary = local.ooniapi_deploy_host + network_name = local.ooniapi_deploy_network + container_port = 80 + host_port_a = local.ooniapi_deploy_ports.ooniprobe[0] + host_port_b = local.ooniapi_deploy_ports.ooniprobe[1] } module "ooniapi_ooniprobe" { @@ -829,6 +858,12 @@ module "ooniapi_reverseproxy_deployer" { secrets = keys(local.ooniapi_deploy_service_secrets.reverseproxy) service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["reverseproxy"].arn deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn + deploy_bucket = aws_s3_bucket.ooniapi_deploy.bucket + deploy_host_primary = local.ooniapi_deploy_host + network_name = local.ooniapi_deploy_network + container_port = 80 + host_port_a = local.ooniapi_deploy_ports.reverseproxy[0] + host_port_b = local.ooniapi_deploy_ports.reverseproxy[1] } module "ooniapi_reverseproxy" { @@ -1147,6 +1182,12 @@ module "ooniapi_oonirun_deployer" { secrets = keys(local.ooniapi_deploy_service_secrets.oonirun) service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["oonirun"].arn deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn + deploy_bucket = aws_s3_bucket.ooniapi_deploy.bucket + deploy_host_primary = local.ooniapi_deploy_host + network_name = local.ooniapi_deploy_network + container_port = 80 + host_port_a = local.ooniapi_deploy_ports.oonirun[0] + host_port_b = local.ooniapi_deploy_ports.oonirun[1] } module "ooniapi_oonirun" { @@ -1209,6 +1250,12 @@ module "ooniapi_oonifindings_deployer" { secrets = keys(local.ooniapi_deploy_service_secrets.oonifindings) service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["oonifindings"].arn deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn + deploy_bucket = aws_s3_bucket.ooniapi_deploy.bucket + deploy_host_primary = local.ooniapi_deploy_host + network_name = local.ooniapi_deploy_network + container_port = 80 + host_port_a = local.ooniapi_deploy_ports.oonifindings[0] + host_port_b = local.ooniapi_deploy_ports.oonifindings[1] } module "ooniapi_oonifindings" { @@ -1285,6 +1332,12 @@ module "ooniapi_ooniauth_deployer" { secrets = keys(local.ooniapi_deploy_service_secrets.ooniauth) service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["ooniauth"].arn deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn + deploy_bucket = aws_s3_bucket.ooniapi_deploy.bucket + deploy_host_primary = local.ooniapi_deploy_host + network_name = local.ooniapi_deploy_network + container_port = 80 + host_port_a = local.ooniapi_deploy_ports.ooniauth[0] + host_port_b = local.ooniapi_deploy_ports.ooniauth[1] } module "ooniapi_ooniauth" { @@ -1371,6 +1424,12 @@ module "ooniapi_oonimeasurements_deployer" { secrets = keys(local.ooniapi_deploy_service_secrets.oonimeasurements) service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["oonimeasurements"].arn deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn + deploy_bucket = aws_s3_bucket.ooniapi_deploy.bucket + deploy_host_primary = local.ooniapi_deploy_host + network_name = local.ooniapi_deploy_network + container_port = 80 + host_port_a = local.ooniapi_deploy_ports.oonimeasurements[0] + host_port_b = local.ooniapi_deploy_ports.oonimeasurements[1] ecs_service_name = module.ooniapi_oonimeasurements.ecs_service_name ecs_cluster_name = module.oonitier1plus_cluster.cluster_name From 8d8baa47981e1539712a2527802ea691ee3ff5bf Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Sun, 4 Oct 2026 05:25:55 +0000 Subject: [PATCH 19/24] ooniapi_gateway: create the deploy user the deploy job logs in as The role gave the deploy user its staging directory and sudoers rule but never created it, so on a fresh host (backend-hel was just re-imaged) its first task owning a path by that user fails, and the deploy job has no account to log in to. The role now creates the user, without groups, and authorizes only ooniapi_gateway_deploy_ssh_public_key, the public half of the key in the environment's oonidevops/ooniapi/deploy_ssh_key secret. It stops with instructions while that is unset; for backend-hel it is left empty until the key pair exists. Run in a Debian bookworm container with the pinned ansible 9.3.0: creates the user and key, a second run changes nothing, and an empty key fails with the message. --- .../backend-hel.ooni.org/ooniapi_gateway.yml | 6 +++++ .../roles/ooniapi_gateway/defaults/main.yml | 5 +++++ ansible/roles/ooniapi_gateway/tasks/main.yml | 22 +++++++++++++++++++ 3 files changed, 33 insertions(+) diff --git a/ansible/host_vars/backend-hel.ooni.org/ooniapi_gateway.yml b/ansible/host_vars/backend-hel.ooni.org/ooniapi_gateway.yml index c47c31ca..227ad4ec 100644 --- a/ansible/host_vars/backend-hel.ooni.org/ooniapi_gateway.yml +++ b/ansible/host_vars/backend-hel.ooni.org/ooniapi_gateway.yml @@ -27,3 +27,9 @@ ooniapi_gateway_services: ssl_domains: - "backend-hel.ooni.org" ooniapi_gateway_cert_ready: false + +# Public half of dev's oonidevops/ooniapi/deploy_ssh_key secret (eu-central-1, +# dev account), which the deploy jobs log in with. Generate the pair, put the +# private key in the secret and the public key here: +# ssh-keygen -t ed25519 -N '' -C ooniapi-deploy-dev -f deploy_dev +ooniapi_gateway_deploy_ssh_public_key: "" diff --git a/ansible/roles/ooniapi_gateway/defaults/main.yml b/ansible/roles/ooniapi_gateway/defaults/main.yml index 6a78c903..64c05696 100644 --- a/ansible/roles/ooniapi_gateway/defaults/main.yml +++ b/ansible/roles/ooniapi_gateway/defaults/main.yml @@ -87,6 +87,11 @@ ooniapi_gateway_nginx_user: www-data # Terraform module. ooniapi_gateway_deploy_user: deploy +# Public half of the key whose private half is in the environment's +# oonidevops/ooniapi/deploy_ssh_key secret, which the deploy job logs in +# with. No default: set per host once that key pair exists. +ooniapi_gateway_deploy_ssh_public_key: "" + # Private directory the deploy job scp's files into before they're # moved/installed into place. Must match STAGING_DIR in # tf/modules/ooniapi_service_deployer/files/deploy.py. diff --git a/ansible/roles/ooniapi_gateway/tasks/main.yml b/ansible/roles/ooniapi_gateway/tasks/main.yml index 8b1594c4..dd099ed8 100644 --- a/ansible/roles/ooniapi_gateway/tasks/main.yml +++ b/ansible/roles/ooniapi_gateway/tasks/main.yml @@ -9,6 +9,28 @@ # Runs as root (become: true at the play level). Each deploy runs `docker # compose up` as root (via the scoped sudoers rule below), matching this. +# The deploy job logs in as this user; it may only run what the sudoers rule +# below allows, so it gets no groups of its own +- name: Check the deploy user's public key is set + ansible.builtin.assert: + that: ooniapi_gateway_deploy_ssh_public_key | length > 0 + fail_msg: >- + Set ooniapi_gateway_deploy_ssh_public_key for {{ inventory_hostname }} + to the public half of the oonidevops/ooniapi/deploy_ssh_key secret. + +- name: Create the deploy user + ansible.builtin.user: + name: "{{ ooniapi_gateway_deploy_user }}" + comment: "ooniapi blue/green deploy job" + shell: /bin/bash + create_home: true + +- name: Authorize the deploy job's key + ansible.posix.authorized_key: + user: "{{ ooniapi_gateway_deploy_user }}" + key: "{{ ooniapi_gateway_deploy_ssh_public_key }}" + exclusive: true + - name: Ensure per-service state directories exist ansible.builtin.file: path: "/etc/ooniapi/{{ item.name }}" From 512bdfb80b80d6ba8c1810cceb62d1ce56941eeb Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Sun, 4 Oct 2026 05:26:47 +0000 Subject: [PATCH 20/24] ooniapi_gateway: run valkey for the services' rate limits oonimeasurements keeps its rate limit counters in valkey, on ECS the ElastiCache Serverless cache, which only answers inside the VPC: from a Hetzner host the service could not reach it. The gateway role now runs a valkey container named valkey on the shared docker network, without a published port, in memory only (save and appendonly off, 256 MB, allkeys-lru): losing the counters on a restart only resets the limits. Dev's oonimeasurements blue/green env uses valkey://valkey:6379; ECS keeps ElastiCache. Each host keeps its own counters, so with two hosts behind DNS a client's limit is per host. Checked with oonimeasurements' limits storage (RedisStorage with async+valkey://) against valkey/valkey:8.1-alpine started with these arguments: a 3/minute limit allows 3 hits and refuses the 4th and 5th. --- ansible/roles/ooniapi_gateway/defaults/main.yml | 4 ++++ ansible/roles/ooniapi_gateway/tasks/main.yml | 17 +++++++++++++++++ tf/environments/dev/main.tf | 2 +- 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/ansible/roles/ooniapi_gateway/defaults/main.yml b/ansible/roles/ooniapi_gateway/defaults/main.yml index 64c05696..26bcc7a6 100644 --- a/ansible/roles/ooniapi_gateway/defaults/main.yml +++ b/ansible/roles/ooniapi_gateway/defaults/main.yml @@ -57,6 +57,10 @@ ooniapi_gateway_testlists_upstream: "" # Docker network shared by every service's containers on this host. ooniapi_gateway_network_name: ooniapi +# Valkey for the services' rate limits, on the shared network as "valkey" +ooniapi_gateway_valkey_image: docker.io/valkey/valkey:8.1-alpine +ooniapi_gateway_valkey_maxmemory: 256mb + # Routes of the API, shared with the ALB (see the file's header); the # gateway's locations are generated from them in tasks/main.yml ooniapi_gateway_routes: "{{ lookup('ansible.builtin.file', role_path ~ '/../../../tf/modules/ooniapi_frontend/routes.yaml') | from_yaml }}" diff --git a/ansible/roles/ooniapi_gateway/tasks/main.yml b/ansible/roles/ooniapi_gateway/tasks/main.yml index dd099ed8..b9721280 100644 --- a/ansible/roles/ooniapi_gateway/tasks/main.yml +++ b/ansible/roles/ooniapi_gateway/tasks/main.yml @@ -212,6 +212,23 @@ ansible.builtin.command: docker network create {{ ooniapi_gateway_network_name }} when: ooniapi_gateway_network_check.rc != 0 +# Rate limit counters of the services on this host (VALKEY_URL +# valkey://valkey:6379), what ElastiCache holds for them on ECS. Reachable +# only on the shared docker network, and kept in memory only: losing the +# counters on a restart resets the limits, as a restart of the services +# would with VALKEY_URL=memory. +- name: Run valkey for the services' rate limits + community.docker.docker_container: + name: valkey + image: "{{ ooniapi_gateway_valkey_image }}" + state: started + restart_policy: always + networks: + - name: "{{ ooniapi_gateway_network_name }}" + command: >- + valkey-server --save "" --appendonly no + --maxmemory {{ ooniapi_gateway_valkey_maxmemory }} --maxmemory-policy allkeys-lru + - name: Ensure sudoers.d directory exists ansible.builtin.file: path: /etc/sudoers.d diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 8192e4f1..5ef6b768 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -1416,7 +1416,7 @@ module "ooniapi_oonimeasurements_deployer" { OTHER_COLLECTORS = jsonencode([for h in local.fastpath_hosts : "http://${h}:8475"]) BASE_URL = "https://api.${local.environment}.ooni.io" S3_BUCKET_NAME = "ooni-data-eu-fra-test" - VALKEY_URL = local.ooniapi_valkey_url + VALKEY_URL = "valkey://valkey:6379" # roles/ooniapi_gateway runs it on the host RATE_LIMITS = "10/minute;400000/day;200000/7day" RATE_LIMITS_WHITELISTED_IPADDRS = jsonencode(["5.9.112.244"]) RATE_LIMITS_UNMETERED_PAGES = jsonencode(["/metrics", "/health"]) From 1e48b4d7322c98ac1771411092b5513337c50c0e Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Sun, 4 Oct 2026 05:27:46 +0000 Subject: [PATCH 21/24] dev: let backend-hel reach fastpath, Postgres and the ClickHouse proxy The dev services on backend-hel reach the same backends as on ECS, but from outside the VPC: - fastpath: the host firewall the fastpath playbook installs already accepts 8472 (measurement submission) and 8475 (measurement spool) from anywhere; it is the instance's security group that only lets in the VPC. ooni_fastpath gains extra_ingress_cidrs (default none, so prod's three instances keep their rules), and dev passes backend-hel. The blue/green env of ooniprobe (FASTPATH_URLS) and oonimeasurements (OTHER_COLLECTORS) uses fastpath.dev.ooni.io instead of the instance's private IP, which only resolves to something reachable inside AWS. - Postgres (publicly accessible, open to 10.0.0.0/8 only) and the ClickHouse proxy (9000-9002, the VPC and listed instances) let backend-hel in too. backend-hel's addresses come from DNS, the way monitoring.ooni.org's do for the ClickHouse proxy. The ClickHouse hosts' own nftables allow list already has backend-hel. Measurements and the spool travel plain HTTP between Hetzner and AWS, as they do from probes. terraform validate and fmt pass for dev and prod. --- tf/environments/dev/main.tf | 22 +++++++++++++++------- tf/modules/ooni_fastpath/main.tf | 4 ++-- tf/modules/ooni_fastpath/variables.tf | 6 ++++++ 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 5ef6b768..7b3dcbde 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -163,9 +163,7 @@ module "oonipg" { db_allocated_storage = "20" db_max_allocated_storage = null - allow_cidr_blocks = [ - "10.0.0.0/8" - ] + allow_cidr_blocks = concat(["10.0.0.0/8"], local.ooniapi_deploy_host_cidrs) allow_security_groups = [module.ooni_jumphost.ec2_sg_id] tags = merge( @@ -350,9 +348,16 @@ resource "aws_secretsmanager_secret_version" "ooniapi_deploy_ssh_key" { # Where the blue/green deploy job puts the dev services: backend-hel alone. # The ports must match ooniapi_gateway_services in # ansible/host_vars/backend-hel.ooni.org/ooniapi_gateway.yml. +data "dns_a_record_set" "ooniapi_deploy_host" { + host = "backend-hel.ooni.org" +} + locals { - ooniapi_deploy_host = "backend-hel.ooni.org" + ooniapi_deploy_host = data.dns_a_record_set.ooniapi_deploy_host.host ooniapi_deploy_network = "ooniapi" + # what the services there connect to from: fastpath, Postgres and the + # ClickHouse proxy let it in, as they let in the VPC + ooniapi_deploy_host_cidrs = [for ip in data.dns_a_record_set.ooniapi_deploy_host.addrs : "${ip}/32"] ooniapi_deploy_ports = { reverseproxy = [18001, 18002] ooniprobe = [18011, 18012] @@ -697,7 +702,7 @@ module "ooniapi_ooniprobe_deployer" { # Pre-wired for the future flip to deploy_mode = "blue_green" env_vars = { FASTPATH_URL = "http://fastpath.${local.environment}.ooni.io:8472" - FASTPATH_URLS = jsonencode([for h in local.fastpath_hosts : "http://${h}:8472"]) + FASTPATH_URLS = jsonencode(["http://fastpath.${local.environment}.ooni.io:8472"]) # private IPs are not reachable from the host FAILED_REPORTS_BUCKET = aws_s3_bucket.ooniprobe_failed_reports.bucket COLLECTOR_ID = 3 # use a different one in prod CONFIG_BUCKET = aws_s3_bucket.ooni_private_config_bucket.bucket @@ -934,7 +939,8 @@ module "ooni_clickhouse_proxy" { to_port = 9002, // for several clickhouse instances protocol = "tcp", cidr_blocks = concat(module.network.vpc_subnet_private[*].cidr_block, ["${module.ooni_fastpath.aws_instance_private_ip}/32", "${module.ooni_fastpath.aws_instance_public_ip}/32"], - ["${module.ooniapi_testlists.aws_instance_private_ip}/32", "${module.ooniapi_testlists.aws_instance_public_ip}/32"]), + ["${module.ooniapi_testlists.aws_instance_private_ip}/32", "${module.ooniapi_testlists.aws_instance_public_ip}/32"], + local.ooniapi_deploy_host_cidrs), }, { // For the prometheus proxy: from_port = 9200, @@ -1133,6 +1139,8 @@ module "ooni_fastpath" { sg_prefix = "oonifastpath" tg_prefix = "fstp" + extra_ingress_cidrs = local.ooniapi_deploy_host_cidrs + monitoring_proxy_private_ip = module.ooni_monitoring_proxy.aws_instance_private_ip monitoring_proxy_public_ip = module.ooni_monitoring_proxy.aws_instance_public_ip @@ -1413,7 +1421,7 @@ module "ooniapi_oonimeasurements_deployer" { # Pre-wired for the future flip to deploy_mode = "blue_green" env_vars = { # it has to be a json-compliant array - OTHER_COLLECTORS = jsonencode([for h in local.fastpath_hosts : "http://${h}:8475"]) + OTHER_COLLECTORS = jsonencode(["http://fastpath.${local.environment}.ooni.io:8475"]) # private IPs are not reachable from the host BASE_URL = "https://api.${local.environment}.ooni.io" S3_BUCKET_NAME = "ooni-data-eu-fra-test" VALKEY_URL = "valkey://valkey:6379" # roles/ooniapi_gateway runs it on the host diff --git a/tf/modules/ooni_fastpath/main.tf b/tf/modules/ooni_fastpath/main.tf index 379224bd..8831ba11 100644 --- a/tf/modules/ooni_fastpath/main.tf +++ b/tf/modules/ooni_fastpath/main.tf @@ -12,13 +12,13 @@ locals { from_port = 8472 to_port = 8472 protocol = "tcp" - cidr_blocks = local.vpc_cidrs + cidr_blocks = concat(local.vpc_cidrs, var.extra_ingress_cidrs) }, { from_port = 8475 # for serving jsonl files to_port = 8475 protocol = "tcp" - cidr_blocks = local.vpc_cidrs + cidr_blocks = concat(local.vpc_cidrs, var.extra_ingress_cidrs) }, { from_port = 9100 diff --git a/tf/modules/ooni_fastpath/variables.tf b/tf/modules/ooni_fastpath/variables.tf index f53ae3ef..dfdfe39b 100644 --- a/tf/modules/ooni_fastpath/variables.tf +++ b/tf/modules/ooni_fastpath/variables.tf @@ -76,3 +76,9 @@ variable "tags" { type = map(string) default = {} } + +variable "extra_ingress_cidrs" { + description = "Sources outside the VPC that may submit measurements (8472) and read the measurement spool (8475), e.g. the hosts the API services are deployed to blue/green." + type = list(string) + default = [] +} From d992421c688f395443941bfff210163fd3433a10 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Sun, 4 Oct 2026 05:41:27 +0000 Subject: [PATCH 22/24] Declare the valkey URL with the other values the host must match oonimeasurements' blue/green env had valkey://valkey:6379 inline, a literal that silently depended on the container name the gateway role picks. It is now ooniapi_deploy_valkey_url, next to the docker network and slot ports in dev's deploy locals, which all have to match the host, and the role's container name is ooniapi_gateway_valkey_name, each pointing at the other. --- ansible/roles/ooniapi_gateway/defaults/main.yml | 4 +++- ansible/roles/ooniapi_gateway/tasks/main.yml | 4 ++-- tf/environments/dev/main.tf | 5 ++++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/ansible/roles/ooniapi_gateway/defaults/main.yml b/ansible/roles/ooniapi_gateway/defaults/main.yml index 26bcc7a6..bd607048 100644 --- a/ansible/roles/ooniapi_gateway/defaults/main.yml +++ b/ansible/roles/ooniapi_gateway/defaults/main.yml @@ -57,7 +57,9 @@ ooniapi_gateway_testlists_upstream: "" # Docker network shared by every service's containers on this host. ooniapi_gateway_network_name: ooniapi -# Valkey for the services' rate limits, on the shared network as "valkey" +# Valkey for the services' rate limits, reached on the shared network by its +# container name: must match ooniapi_deploy_valkey_url in tf/environments +ooniapi_gateway_valkey_name: valkey ooniapi_gateway_valkey_image: docker.io/valkey/valkey:8.1-alpine ooniapi_gateway_valkey_maxmemory: 256mb diff --git a/ansible/roles/ooniapi_gateway/tasks/main.yml b/ansible/roles/ooniapi_gateway/tasks/main.yml index b9721280..156ba89c 100644 --- a/ansible/roles/ooniapi_gateway/tasks/main.yml +++ b/ansible/roles/ooniapi_gateway/tasks/main.yml @@ -213,13 +213,13 @@ when: ooniapi_gateway_network_check.rc != 0 # Rate limit counters of the services on this host (VALKEY_URL -# valkey://valkey:6379), what ElastiCache holds for them on ECS. Reachable +# valkey://:6379), what ElastiCache holds for them on ECS. Reachable # only on the shared docker network, and kept in memory only: losing the # counters on a restart resets the limits, as a restart of the services # would with VALKEY_URL=memory. - name: Run valkey for the services' rate limits community.docker.docker_container: - name: valkey + name: "{{ ooniapi_gateway_valkey_name }}" image: "{{ ooniapi_gateway_valkey_image }}" state: started restart_policy: always diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 7b3dcbde..4ac93180 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -355,6 +355,9 @@ data "dns_a_record_set" "ooniapi_deploy_host" { locals { ooniapi_deploy_host = data.dns_a_record_set.ooniapi_deploy_host.host ooniapi_deploy_network = "ooniapi" + # the valkey container roles/ooniapi_gateway runs on that network + # (ooniapi_gateway_valkey_name), for the services' rate limits + ooniapi_deploy_valkey_url = "valkey://valkey:6379" # what the services there connect to from: fastpath, Postgres and the # ClickHouse proxy let it in, as they let in the VPC ooniapi_deploy_host_cidrs = [for ip in data.dns_a_record_set.ooniapi_deploy_host.addrs : "${ip}/32"] @@ -1424,7 +1427,7 @@ module "ooniapi_oonimeasurements_deployer" { OTHER_COLLECTORS = jsonencode(["http://fastpath.${local.environment}.ooni.io:8475"]) # private IPs are not reachable from the host BASE_URL = "https://api.${local.environment}.ooni.io" S3_BUCKET_NAME = "ooni-data-eu-fra-test" - VALKEY_URL = "valkey://valkey:6379" # roles/ooniapi_gateway runs it on the host + VALKEY_URL = local.ooniapi_deploy_valkey_url RATE_LIMITS = "10/minute;400000/day;200000/7day" RATE_LIMITS_WHITELISTED_IPADDRS = jsonencode(["5.9.112.244"]) RATE_LIMITS_UNMETERED_PAGES = jsonencode(["/metrics", "/health"]) From 765c7648a046a7b9ce53dc680cf8df7fc6a9a63f Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Sun, 4 Oct 2026 05:53:42 +0000 Subject: [PATCH 23/24] Don't give ooniauth and oonimeasurements the Postgres URL on the hosts Neither service uses Postgres: ooniauth sends login emails through SES and signs JWTs, oonimeasurements reads ClickHouse only. Their blue/green service secrets still carried POSTGRESQL_URL, copied from their ECS task_secrets, and the deploy job would write it to disk on the host. Removed from dev's service secrets and from prod's commented-out ones. prune_service_credentials removes the same variable from their ECS tasks. terraform validate passes for dev and prod. --- tf/environments/dev/main.tf | 2 -- tf/environments/prod/main.tf | 2 -- 2 files changed, 4 deletions(-) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 4ac93180..f19f259d 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -297,7 +297,6 @@ locals { PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value } ooniauth = { - POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value ACCOUNT_ID_HASHING_KEY = data.aws_ssm_parameter.account_id_hashing_key.value @@ -305,7 +304,6 @@ locals { AWS_ACCESS_KEY_ID = data.aws_secretsmanager_secret_version.ooniapi_user_access_key_id.secret_string } oonimeasurements = { - POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_oonimeasurements_test_url.value diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf index 5a544366..a02d6c7e 100644 --- a/tf/environments/prod/main.tf +++ b/tf/environments/prod/main.tf @@ -334,7 +334,6 @@ resource "aws_secretsmanager_secret_version" "oonipg_url" { # PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value # } # ooniauth = { -# POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value # JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value # PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value # ACCOUNT_ID_HASHING_KEY = data.aws_ssm_parameter.account_id_hashing_key.value @@ -342,7 +341,6 @@ resource "aws_secretsmanager_secret_version" "oonipg_url" { # AWS_ACCESS_KEY_ID = data.aws_secretsmanager_secret_version.ooniapi_user_access_key_id.secret_string # } # oonimeasurements = { -# POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value # JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value # PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value # CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_oonimeasurements_url.value From 699c8ce2f09e18db1c7825e040cb615b27b13f9b Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Sun, 4 Oct 2026 06:37:19 +0000 Subject: [PATCH 24/24] ooniapi_gateway: key cached responses on the Origin header too The services' CORS middleware (Starlette) answers a request from an allowed origin with Access-Control-Allow-Origin set to that origin, and a request without an Origin, or from a disallowed one, with no CORS headers and no Vary: Origin. The cache key left Origin out, so whichever request filled an entry decided what every origin got: one curl request without Origin was enough to make browsers on the Explorer fail the response's CORS check, for up to a day in the long zone. The key now includes $http_origin, so each origin gets the response the service gave it. A client sending random origins only misses the cache, as random query parameters already would. Reproduced with the gateway rendered for backend-hel in nginx 1.27, in front of a FastAPI app with oonimeasurements' CORS settings, on /api/v1/aggregation: - before: a request without Origin, then one from explorer.ooni.org: HIT without Access-Control-Allow-Origin - after: the explorer.ooni.org request is a MISS answered with its Access-Control-Allow-Origin, and a HIT with it when repeated; an unknown origin gets no CORS header; test-lists.ooni.org gets its own --- ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 b/ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 index a6563a5b..7a584f24 100644 --- a/ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 +++ b/ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 @@ -77,7 +77,11 @@ server { # without one. Requests with credentials never use the cache. Each cached # location keys on its service's $ooniapi_cache_version_, the # image tag its blue/green deploy writes to -upstream.conf, so a - # deploy invalidates that service's responses, and only those. + # deploy invalidates that service's responses, and only those. The key + # includes the Origin header: the services' CORS middleware answers an + # allowed origin with its own Access-Control-Allow-Origin, and a request + # without one (or with another) without CORS headers or Vary, so a + # response cached for one origin must not be served to another. proxy_cache_methods GET HEAD; proxy_cache_valid 200 301 302 {{ ooniapi_gateway_cache_default_valid }}; proxy_cache_valid any 0; @@ -93,7 +97,7 @@ server { ## --- generated from routes.yaml, see tasks/main.yml --- {% for l in ooniapi_gateway_locations %} {% if l.service in service_names %} - location {{ l.match }} {{ l.path }} { proxy_pass http://{{ l.service }};{% if l.cache %} proxy_cache ooniapi_{{ l.cache }}; proxy_cache_key "$ooniapi_cache_version_{{ l.service }}$scheme$host$request_uri";{% endif %} } # {{ l.route }} + location {{ l.match }} {{ l.path }} { proxy_pass http://{{ l.service }};{% if l.cache %} proxy_cache ooniapi_{{ l.cache }}; proxy_cache_key "$ooniapi_cache_version_{{ l.service }}|$http_origin|$scheme$host$request_uri";{% endif %} } # {{ l.route }} {% elif l.service in remote_upstreams %} # {{ l.service }} is not yet migrated off AWS, proxied cross-cloud until it is location {{ l.match }} {{ l.path }} { proxy_pass {{ remote_upstreams[l.service] }}; } # {{ l.route }}