diff --git a/ansible/deploy-ooniapi-gateway.yml b/ansible/deploy-ooniapi-gateway.yml new file mode 100644 index 00000000..f65c03a9 --- /dev/null +++ b/ansible/deploy-ooniapi-gateway.yml @@ -0,0 +1,7 @@ +--- +- name: Bootstrap the tier0 blue/green gateway on dedicated backend hosts + hosts: backend-fsn.ooni.org:backend-hel.ooni.org + become: true + roles: + - role: ooniapi_gateway + tags: ooniapi_gateway diff --git a/ansible/roles/ooniapi_gateway/defaults/main.yml b/ansible/roles/ooniapi_gateway/defaults/main.yml new file mode 100644 index 00000000..0b28f33b --- /dev/null +++ b/ansible/roles/ooniapi_gateway/defaults/main.yml @@ -0,0 +1,68 @@ +--- +# Domain the gateway vhost answers on for path-based routing. Mirrors the +# ALB's `ooniapi_frontend_main_domain_name` in +# tf/modules/ooniapi_frontend/main.tf (e.g. "api.prod.ooni.io"). +# No default: set explicitly per host in host_vars. +ooniapi_gateway_primary_domain: "" + +# Any additional hostnames that should hit the same path-routed vhost. +# Mirrors extra entries in `ooniapi_frontend_alternative_domains` that +# aren't handled elsewhere (e.g. "api.ooni.org" in prod). +ooniapi_gateway_extra_server_names: [] + +# Suffix used for the ALB's secondary, host-header-only routing rules, e.g. +# "oonirun.prod.ooni.io". Mirrors `direct_domain_suffix` in +# tf/modules/ooniapi_frontend/main.tf ("${stage}.ooni.io"). +# No default: set explicitly per host in host_vars. +ooniapi_gateway_direct_domain_suffix: "" + +# Name of the dehydrated cert (the directory under +# ooniapi_gateway_cert_path) that covers ooniapi_gateway_primary_domain, +# every ".{{ ooniapi_gateway_direct_domain_suffix }}" name, and +# ooniapi_gateway_extra_server_names as SANs. +# +# This role does NOT manage that cert -- issuing it means extending (or +# adding to) the `dehydrated` role's ssl_domains on this host, which in turn +# requires DNS for every one of those names to already resolve here. That's +# a DNS-cutover-time step, not a day-1 bootstrap step, so it's deliberately +# left manual. See the role README notes. +ooniapi_gateway_cert_name: "{{ ooniapi_gateway_primary_domain }}" +ooniapi_gateway_cert_path: /var/lib/dehydrated/certs/ + +# Flip to true only once ooniapi_gateway_cert_name actually exists on disk. +# Until then the vhost file is rendered but not installed, so this role is +# safe to run ahead of the real DNS cutover. +ooniapi_gateway_cert_ready: false + +# Services fronted by this gateway, and the two host ports their blue/green +# slots are bound to. host_port_a/host_port_b must match the values passed +# to that service's ooniapi_service_deployer module invocation in Terraform. +# +# Example: +# ooniapi_gateway_services: +# - name: reverseproxy +# host_port_a: 18001 +# host_port_b: 18002 +# - name: oonirun +# host_port_a: 18011 +# host_port_b: 18012 +ooniapi_gateway_services: [] + +# Cross-cloud passthrough for tier0 services not yet migrated off AWS +# (currently just testlists). Set to its public hostname to proxy those +# paths there until testlists itself is migrated; leave empty to omit +# those routes entirely. +ooniapi_gateway_testlists_upstream: "" + +# Docker network shared by every service's containers on this host. +ooniapi_gateway_network_name: ooniapi + +# SSH user the deploy CodeBuild job connects as. Must match +# `deploy_ssh_user` (default "deploy") in the ooniapi_service_deployer +# Terraform module. +ooniapi_gateway_deploy_user: deploy + +# Private directory the deploy job scp's files into before they're +# moved/installed into place. Must match STAGING_DIR in +# tf/modules/ooniapi_service_deployer/files/deploy.py. +ooniapi_gateway_staging_dir: /var/lib/ooniapi/deploy-staging diff --git a/ansible/roles/ooniapi_gateway/handlers/main.yml b/ansible/roles/ooniapi_gateway/handlers/main.yml new file mode 100644 index 00000000..9d61904d --- /dev/null +++ b/ansible/roles/ooniapi_gateway/handlers/main.yml @@ -0,0 +1,14 @@ +--- +- name: test ooniapi gateway nginx config + ansible.builtin.command: /usr/sbin/nginx -t -c /etc/nginx/nginx.conf + listen: reload ooniapi gateway nginx + +- name: reload ooniapi gateway nginx + ansible.builtin.service: + name: nginx + state: reloaded + +- name: restart docker + ansible.builtin.systemd_service: + name: docker + state: restarted diff --git a/ansible/roles/ooniapi_gateway/tasks/main.yml b/ansible/roles/ooniapi_gateway/tasks/main.yml new file mode 100644 index 00000000..020790d3 --- /dev/null +++ b/ansible/roles/ooniapi_gateway/tasks/main.yml @@ -0,0 +1,227 @@ +--- +# One-time (and idempotent-on-rerun) bootstrap for a host that will run +# tier0 services via Docker Compose blue/green deploys, plus the nginx +# gateway vhost that fronts them. See the Terraform side of this in +# tf/modules/ooniapi_service_deployer (deploy_mode = "blue_green") and its +# files/deploy.py, which is what actually performs each deploy -- this role +# only prepares the host for that job to run against. +# +# Runs as root (become: true at the play level). Each deploy runs `docker +# compose up` as root (via the scoped sudoers rule below), matching this. + +- name: Ensure per-service state directories exist + ansible.builtin.file: + path: "/etc/ooniapi/{{ item.name }}" + state: directory + owner: root + group: root + mode: "0755" + loop: "{{ ooniapi_gateway_services }}" + loop_control: + label: "{{ item.name }}" + +# Files land here (not shared /tmp) before being moved/installed into +# place, so the per-secret files scp'd in on every deploy are never briefly +# world-readable on their way in -- only the deploy user (and root) can +# even list this directory. +- name: Ensure the deploy user's private staging directory exists + ansible.builtin.file: + path: "{{ ooniapi_gateway_staging_dir }}" + state: directory + owner: "{{ ooniapi_gateway_deploy_user }}" + group: "{{ ooniapi_gateway_deploy_user }}" + mode: "0700" + +- name: Check whether each service already has an active_slot marker + ansible.builtin.stat: + path: "/etc/ooniapi/{{ item.name }}/active_slot" + loop: "{{ ooniapi_gateway_services }}" + loop_control: + label: "{{ item.name }}" + register: ooniapi_gateway_active_slot_stat + +- name: Seed active_slot with "a" for services deploying for the first time + ansible.builtin.copy: + dest: "/etc/ooniapi/{{ item.item.name }}/active_slot" + content: "a" + owner: root + group: root + mode: "0644" + loop: "{{ ooniapi_gateway_active_slot_stat.results }}" + loop_control: + label: "{{ item.item.name }}" + when: not item.stat.exists + +- name: Check whether each service already has an nginx upstream conf + ansible.builtin.stat: + path: "/etc/nginx/conf.d/{{ item.name }}-upstream.conf" + loop: "{{ ooniapi_gateway_services }}" + loop_control: + label: "{{ item.name }}" + register: ooniapi_gateway_upstream_stat + +# Seeded with slot "a" up / "b" down, matching active_slot's "a" default +# above, so the very first `nginx -t` (run when installing the gateway +# vhost below) succeeds before that service's first real deploy has run. +- name: Seed a placeholder upstream conf for services deploying for the first time + ansible.builtin.copy: + dest: "/etc/nginx/conf.d/{{ item.item.name }}-upstream.conf" + content: | + upstream {{ item.item.name }} { + server 127.0.0.1:{{ item.item.host_port_a }}; + server 127.0.0.1:{{ item.item.host_port_b }} down; + } + owner: root + group: root + mode: "0644" + loop: "{{ ooniapi_gateway_upstream_stat.results }}" + loop_control: + label: "{{ item.item.name }}" + when: not item.stat.exists + notify: reload ooniapi gateway nginx + +# Follows https://docs.docker.com/engine/install/debian/ (the officially +# supported docker-ce packages, rather than Debian's own docker.io/ +# docker-compose-v2) so we get upstream's release cadence. The repo URL is +# built from ansible_distribution_release rather than a hardcoded codename +# (e.g. "trixie") so this keeps working unchanged across Debian releases. +- name: Remove conflicting docker packages + ansible.builtin.apt: + name: + - docker.io + - docker-doc + - docker-compose + - docker-compose-v2 + - podman-docker + - containerd + - runc + state: absent + +- name: Install docker's prerequisites + ansible.builtin.apt: + name: + - ca-certificates + - curl + - gnupg + state: present + update_cache: true + +- name: Ensure the apt keyrings directory exists + ansible.builtin.file: + path: /etc/apt/keyrings + state: directory + mode: "0755" + +- name: Download docker's official apt signing key + ansible.builtin.get_url: + url: "https://download.docker.com/linux/{{ ansible_distribution | lower }}/gpg" + dest: /etc/apt/keyrings/docker.asc + mode: "0644" + +- name: Read the fingerprint of the downloaded key + ansible.builtin.command: gpg --with-colons --show-keys /etc/apt/keyrings/docker.asc + register: ooniapi_gateway_docker_key_check + changed_when: false + +# Pins against docker's known-good key fingerprint (9DC8 5822 9FC7 DD38 +# 854A E2D8 8D81 803C 0EBF CD88) so a compromised/MITM'd download of the key +# itself (which apt would otherwise trust blindly via signed-by=) gets +# caught here instead of silently being trusted for every future `apt +# update`. +- name: Verify the downloaded key's fingerprint matches docker's published one + ansible.builtin.assert: + that: + - ooniapi_gateway_docker_key_check.stdout is + search('^fpr:::::::::9DC858229FC7DD38854AE2D88D81803C0EBFCD88:$', multiline=true) + fail_msg: >- + docker.asc's fingerprint does not match the expected + 9DC858229FC7DD38854AE2D88D81803C0EBFCD88 -- refusing to trust it. + success_msg: "docker.asc's fingerprint matches the expected value." + +- name: Get dpkg architecture + ansible.builtin.command: dpkg --print-architecture + register: ooniapi_gateway_dpkg_arch + changed_when: false + +- name: Add the docker apt repository + ansible.builtin.apt_repository: + repo: >- + deb [arch={{ ooniapi_gateway_dpkg_arch.stdout }} signed-by=/etc/apt/keyrings/docker.asc] + https://download.docker.com/linux/{{ ansible_distribution | lower }} + {{ ansible_distribution_release }} stable + filename: docker + state: present + +- name: Install docker + ansible.builtin.apt: + name: + - docker-ce + - docker-ce-cli + - containerd.io + - docker-compose-plugin + state: present + update_cache: true + +- name: Make docker's containerd use the nftables firewall backend + ansible.builtin.lineinfile: + path: /etc/containerd/config.toml + line: 'firewall-backend="nftables"' + state: present + create: true + mode: "0644" + notify: restart docker + +- name: Ensure docker is running + ansible.builtin.systemd_service: + name: docker + state: started + enabled: true + +- name: Check whether the shared docker network already exists + ansible.builtin.command: docker network inspect {{ ooniapi_gateway_network_name }} + register: ooniapi_gateway_network_check + changed_when: false + failed_when: false + +- name: Create the shared docker network + ansible.builtin.command: docker network create {{ ooniapi_gateway_network_name }} + when: ooniapi_gateway_network_check.rc != 0 + +- name: Ensure sudoers.d directory exists + ansible.builtin.file: + path: /etc/sudoers.d + state: directory + owner: root + group: root + +- name: Install scoped sudoers rule for the deploy user + ansible.builtin.template: + src: sudoers-ooniapi-deploy.j2 + dest: /etc/sudoers.d/90-ooniapi-deploy + owner: root + group: root + mode: "0440" + validate: "visudo -cf %s" + +- name: Render the gateway vhost + ansible.builtin.template: + src: gateway.conf.j2 + dest: /etc/nginx/conf.d/ooniapi-gateway.conf.pending + owner: root + group: root + mode: "0644" + +# Split into "render" + "install" so a bad template is visible +# (.conf.pending) without ever being loaded by nginx, and so the vhost is +# never installed at all until ooniapi_gateway_cert_ready is true -- see +# defaults/main.yml for why that's a separate, deliberate step. +- name: Install the gateway vhost now that its cert is ready + ansible.builtin.copy: + remote_src: true + src: /etc/nginx/conf.d/ooniapi-gateway.conf.pending + dest: /etc/nginx/conf.d/ooniapi-gateway.conf + owner: root + group: root + mode: "0644" + when: ooniapi_gateway_cert_ready + notify: reload ooniapi gateway nginx diff --git a/ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 b/ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 new file mode 100644 index 00000000..e87e46ef --- /dev/null +++ b/ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 @@ -0,0 +1,167 @@ +# Managed by ansible - roles/ooniapi_gateway/templates/gateway.conf.j2 +# +# Single "gateway" vhost for the OONI tier0 API on this dedicated host, +# mirroring (1:1, by design) the AWS ALB listener rules in +# tf/modules/ooniapi_frontend/main.tf. Only rules for services actually +# running blue/green here ({{ ooniapi_gateway_services | map(attribute='name') | join(', ') }}) +# are handled; everything unmatched falls through to "reverseproxy", exactly +# like the ALB's default_action. +# +# Each service's own `-upstream.conf` (the `upstream { +# ... }` block referenced below via proxy_pass) is dropped into this same +# conf.d directory by that service's blue/green deploy job, not by this +# role -- see ooniapi_service_deployer's buildspec_deploy.yml. A service's +# location block here will fail nginx -t until that service has deployed at +# least once. + +{% set service_names = ooniapi_gateway_services | map(attribute='name') | list %} + +# anonymize ipaddr (same scheme as the legacy ooni-api vhost) +map $remote_addr $ooniapi_gateway_remote_addr_anon { + ~(?P\d+\.\d+\.\d+)\. $ip.0; + ~(?P[^:]+:[^:]+): $ip::; + default 0.0.0.0; +} + +log_format ooniapi_gateway_fmt '$ooniapi_gateway_remote_addr_anon [$time_local] ' + '"$request" $status $body_bytes_sent rt:$request_time "$http_referer" "$http_user_agent"'; + +server { + listen 443 ssl http2; + listen [::]:443 ssl http2; + server_name {{ ([ooniapi_gateway_primary_domain] + ooniapi_gateway_extra_server_names) | join(' ') }}; + + access_log syslog:server=unix:/dev/log,tag=ooniapi-gateway,severity=info ooniapi_gateway_fmt; + error_log syslog:server=unix:/dev/log,tag=ooniapi-gateway,severity=info; + + client_max_body_size 200M; # for measurement POST, matches the legacy API vhost + + ssl_certificate {{ ooniapi_gateway_cert_path }}{{ ooniapi_gateway_cert_name }}/fullchain.pem; + ssl_certificate_key {{ ooniapi_gateway_cert_path }}{{ ooniapi_gateway_cert_name }}/privkey.pem; + ssl_trusted_certificate {{ ooniapi_gateway_cert_path }}{{ ooniapi_gateway_cert_name }}/chain.pem; + + # https://ssl-config.mozilla.org/#server=nginx&config=intermediate -- kept + # inline (rather than shared include) so this role has no dependency on + # how nginx itself got installed on this host. + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384; + ssl_prefer_server_ciphers off; + ssl_session_timeout 5m; + ssl_session_cache shared:MozSSL:30m; + ssl_session_tickets off; + ssl_stapling on; + ssl_stapling_verify on; + + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; + add_header X-Frame-Options DENY always; + add_header X-Content-Type-Options nosniff always; + + proxy_http_version 1.1; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 900; + +{% if 'ooniauth' in service_names %} + ## --- ooniauth --- (ALB priority 108) + location ^~ /api/v2/ooniauth/ { proxy_pass http://ooniauth; } + location = /api/v1/user_register { proxy_pass http://ooniauth; } + location = /api/v1/user_login { proxy_pass http://ooniauth; } + location = /api/v1/user_refresh_token { proxy_pass http://ooniauth; } + location = /api/_/account_metadata { proxy_pass http://ooniauth; } +{% endif %} + +{% if 'oonirun' in service_names %} + ## --- oonirun --- (ALB priority 110) + location ^~ /api/v2/oonirun/ { proxy_pass http://oonirun; } +{% endif %} + +{% if 'ooniprobe' in service_names %} + ## --- ooniprobe --- (ALB priorities 120-123) + location ^~ /api/v2/ooniprobe/ { proxy_pass http://ooniprobe; } + location = /api/v1/login { proxy_pass http://ooniprobe; } + location = /api/v1/register { proxy_pass http://ooniprobe; } + location ^~ /api/v1/update/ { proxy_pass http://ooniprobe; } + location ^~ /api/v1/check-in { proxy_pass http://ooniprobe; } + location ^~ /api/v1/test-helpers { proxy_pass http://ooniprobe; } + location = /api/v1/test-list/urls { proxy_pass http://ooniprobe; } + location ^~ /report { proxy_pass http://ooniprobe; } + location = /api/_/show_countries_prioritization { proxy_pass http://ooniprobe; } + location = /api/_/debug_prioritization { proxy_pass http://ooniprobe; } + location ^~ /api/v1/manifest { proxy_pass http://ooniprobe; } + location ^~ /api/v1/sign_credential { proxy_pass http://ooniprobe; } + location ^~ /api/v1/submit_measurement { proxy_pass http://ooniprobe; } + location ^~ /bouncer/net-tests { proxy_pass http://ooniprobe; } + location ^~ /api/v1/geolookup { proxy_pass http://ooniprobe; } + location ^~ /api/v1/collectors { proxy_pass http://ooniprobe; } + location = /api/v1/test-list/tor-targets { proxy_pass http://ooniprobe; } + location = /api/v1/test-list/psiphon-config { proxy_pass http://ooniprobe; } +{% endif %} + +{% if 'oonifindings' in service_names %} + ## --- oonifindings --- (ALB priority 130) + location ^~ /api/v1/incidents/ { proxy_pass http://oonifindings; } +{% endif %} + +{% if 'oonimeasurements' in service_names %} + ## --- oonimeasurements --- (ALB priorities 140-143) + location ^~ /api/v1/measurements/ { proxy_pass http://oonimeasurements; } + location = /api/v1/raw_measurement { proxy_pass http://oonimeasurements; } + location = /api/v1/measurement_meta { proxy_pass http://oonimeasurements; } + location = /api/v1/measurements { proxy_pass http://oonimeasurements; } + location = /api/v1/torsf_stats { proxy_pass http://oonimeasurements; } + location = /api/v1/aggregation { proxy_pass http://oonimeasurements; } + location ^~ /api/v1/aggregation/ { proxy_pass http://oonimeasurements; } + location = /api/v1/observations { proxy_pass http://oonimeasurements; } + location = /api/v1/analysis { proxy_pass http://oonimeasurements; } + location = /api/v1/detector/changepoints { proxy_pass http://oonimeasurements; } +{% endif %} + +{% if ooniapi_gateway_testlists_upstream %} + ## --- testlists (ALB priority 144; not yet migrated off AWS, proxied + ## cross-cloud until it is) --- + location ^~ /api/_/url-submission/test-list/ { proxy_pass https://{{ ooniapi_gateway_testlists_upstream }}; } + location = /api/_/url-priorities/list { proxy_pass https://{{ ooniapi_gateway_testlists_upstream }}; } + location = /api/_/url-priorities/update { proxy_pass https://{{ ooniapi_gateway_testlists_upstream }}; } + location = /api/v1/url-submission/submit { proxy_pass https://{{ ooniapi_gateway_testlists_upstream }}; } + location = /api/v1/url-submission/update-url { proxy_pass https://{{ ooniapi_gateway_testlists_upstream }}; } +{% endif %} + + ## --- default: everything else goes to reverseproxy, same as the ALB's + ## default_action. Covers legacy/unmigrated paths (oonith is NOT among + ## them -- it's routed by host_header on *.th.ooni.org, a hostname this + ## vhost doesn't answer for, so it never reaches here either way). --- + location / { + proxy_pass http://reverseproxy; + } +} + +{% for service in service_names %} +{% if service != 'reverseproxy' %} +server { + listen 443 ssl http2; + listen [::]:443 ssl http2; + server_name {{ service }}.{{ ooniapi_gateway_direct_domain_suffix }}; + + access_log syslog:server=unix:/dev/log,tag=ooniapi-gateway,severity=info ooniapi_gateway_fmt; + error_log syslog:server=unix:/dev/log,tag=ooniapi-gateway,severity=info; + + ssl_certificate {{ ooniapi_gateway_cert_path }}{{ ooniapi_gateway_cert_name }}/fullchain.pem; + ssl_certificate_key {{ ooniapi_gateway_cert_path }}{{ ooniapi_gateway_cert_name }}/privkey.pem; + ssl_trusted_certificate {{ ooniapi_gateway_cert_path }}{{ ooniapi_gateway_cert_name }}/chain.pem; + ssl_protocols TLSv1.2 TLSv1.3; + + proxy_http_version 1.1; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 900; + + location / { + proxy_pass http://{{ service }}; + } +} +{% endif %} +{% endfor %} diff --git a/ansible/roles/ooniapi_gateway/templates/sudoers-ooniapi-deploy.j2 b/ansible/roles/ooniapi_gateway/templates/sudoers-ooniapi-deploy.j2 new file mode 100644 index 00000000..dacbb55c --- /dev/null +++ b/ansible/roles/ooniapi_gateway/templates/sudoers-ooniapi-deploy.j2 @@ -0,0 +1,29 @@ +# Managed by ansible - roles/ooniapi_gateway/templates/sudoers-ooniapi-deploy.j2 +# +# Scoped to exactly the remote commands issued by +# tf/modules/ooniapi_service_deployer/files/deploy.py over SSH as +# {{ ooniapi_gateway_deploy_user }}. Intentionally narrower than the blanket +# "NOPASSWD:ALL" pattern used for the deploy user in other roles (e.g. +# roles/anonc) -- this user is driven by an automated CI job over the +# network, not a human operator. +# +# Sources below are always files this same job just scp'd into +# {{ ooniapi_gateway_staging_dir }}, a directory only {{ ooniapi_gateway_deploy_user }} +# (and root) can read -- see tasks/main.yml for why that, and not shared +# /tmp, is where the per-secret files (each a Compose file-based secret, +# mounted read-only into its container -- never an environment variable) +# land. +# +# Caveat: sudoers glob matching (fnmatch without FNM_PATHNAME) lets "*" +# match "/", so this is a theoretical rather than practical widening of +# scope, but worth knowing if you're auditing this file. + +Cmnd_Alias OONIAPI_DEPLOY_MV_COMPOSE = /usr/bin/mv {{ ooniapi_gateway_staging_dir }}/*.yaml /etc/ooniapi/*/*.yaml +Cmnd_Alias OONIAPI_DEPLOY_MV_UPSTREAM = /usr/bin/mv {{ ooniapi_gateway_staging_dir }}/*-upstream.conf /etc/nginx/conf.d/*-upstream.conf +Cmnd_Alias OONIAPI_DEPLOY_SECRET_INSTALL = /usr/bin/install -D -m 600 -o root -g root {{ ooniapi_gateway_staging_dir }}/*.secret /etc/ooniapi/*/*/secrets/* +Cmnd_Alias OONIAPI_DEPLOY_COMPOSE_UP = /usr/bin/docker compose -f /etc/ooniapi/*/*.yaml up -d --pull always --remove-orphans +Cmnd_Alias OONIAPI_DEPLOY_NGINX_RELOAD = /usr/bin/systemctl reload nginx +Cmnd_Alias OONIAPI_DEPLOY_NGINX_TEST = /usr/sbin/nginx -t +Cmnd_Alias OONIAPI_DEPLOY_MARKER = /usr/bin/tee /etc/ooniapi/*/active_slot + +{{ ooniapi_gateway_deploy_user }} ALL=(root) NOPASSWD: OONIAPI_DEPLOY_MV_COMPOSE, OONIAPI_DEPLOY_MV_UPSTREAM, OONIAPI_DEPLOY_SECRET_INSTALL, OONIAPI_DEPLOY_COMPOSE_UP, OONIAPI_DEPLOY_NGINX_RELOAD, OONIAPI_DEPLOY_NGINX_TEST, OONIAPI_DEPLOY_MARKER diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 5d1f9e64..31805934 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -253,6 +253,97 @@ data "aws_ssm_parameter" "account_id_hashing_key" { name = "/oonidevops/secrets/ooni_services/account_id_hashing_key" } +### Blue/green deploy secrets (Docker + systemd on dedicated Hetzner hosts) +# +# These mirror the values already passed as `task_secrets` to the ECS task +# definitions above, but consolidated into one JSON blob per service so the +# "blue_green" deploy_mode CodeBuild job can fetch them in a single +# secretsmanager:GetSecretValue call and write each key into the target +# slot's env file. Values are pulled from the same underlying data sources +# used by the ECS `task_secrets` maps, so both deploy paths stay in sync +# automatically. + +data "aws_secretsmanager_secret_version" "ooniapi_user_access_key_id" { + secret_id = module.ooniapi_user.aws_access_key_id_arn +} + +data "aws_secretsmanager_secret_version" "ooniapi_user_secret_access_key" { + secret_id = module.ooniapi_user.aws_secret_access_key_arn +} + +locals { + ooniapi_deploy_service_secrets = { + reverseproxy = { + PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value + } + ooniprobe = { + POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value + JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret_legacy.value + PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_write_url.value + ANONC_SECRET_KEY = data.aws_ssm_parameter.anonc_secret_key.value + } + oonirun = { + POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value + JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value + PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_readonly_url.value + } + oonifindings = { + POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value + JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value + PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_readonly_url.value + } + ooniauth = { + POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value + JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value + PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value + ACCOUNT_ID_HASHING_KEY = data.aws_ssm_parameter.account_id_hashing_key.value + AWS_SECRET_ACCESS_KEY = data.aws_secretsmanager_secret_version.ooniapi_user_secret_access_key.secret_string + AWS_ACCESS_KEY_ID = data.aws_secretsmanager_secret_version.ooniapi_user_access_key_id.secret_string + } + oonimeasurements = { + POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value + JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value + PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_readonly_test_url.value + ACCOUNT_ID_HASHING_KEY = data.aws_ssm_parameter.account_id_hashing_key.value + } + } +} + +resource "aws_secretsmanager_secret" "ooniapi_deploy_service_secrets" { + for_each = local.ooniapi_deploy_service_secrets + + name = "oonidevops/ooniapi/${each.key}/service_secrets" + tags = local.tags +} + +resource "aws_secretsmanager_secret_version" "ooniapi_deploy_service_secrets" { + for_each = local.ooniapi_deploy_service_secrets + + secret_id = aws_secretsmanager_secret.ooniapi_deploy_service_secrets[each.key].id + secret_string = jsonencode(each.value) +} + +# Shared by every service's blue/green deploy job. The private key itself is +# generated and rotated out-of-band (see the deploy README); Terraform only +# owns the secret container, not its value. +resource "aws_secretsmanager_secret" "ooniapi_deploy_ssh_key" { + name = "oonidevops/ooniapi/deploy_ssh_key" + tags = local.tags +} + +resource "aws_secretsmanager_secret_version" "ooniapi_deploy_ssh_key" { + secret_id = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.id + secret_string = "REPLACE_ME: populate out-of-band with the \"deploy\" user's SSH private key" + + lifecycle { + ignore_changes = [secret_string] + } +} + resource "random_id" "artifact_id" { byte_length = 4 } @@ -557,6 +648,10 @@ EOF module "ooniapi_ooniprobe_deployer" { source = "../../modules/ooniapi_service_deployer" + # Flip to "blue_green" to switch this service to the Docker + systemd + # blue/green deploy on the dedicated Hetzner hosts. + deploy_mode = "ecs" + service_name = "ooniprobe" repo = "ooni/backend" branch_name = "master" @@ -569,6 +664,22 @@ module "ooniapi_ooniprobe_deployer" { ecs_service_name = module.ooniapi_ooniprobe.ecs_service_name ecs_cluster_name = module.ooniapi_cluster.cluster_name + + # Pre-wired for the future flip to deploy_mode = "blue_green" + env_vars = { + FASTPATH_URL = "http://fastpath.${local.environment}.ooni.io:8472" + FASTPATH_URLS = jsonencode([for h in local.fastpath_hosts : "http://${h}:8472"]) + FAILED_REPORTS_BUCKET = aws_s3_bucket.ooniprobe_failed_reports.bucket + COLLECTOR_ID = 3 # use a different one in prod + CONFIG_BUCKET = aws_s3_bucket.ooni_private_config_bucket.bucket + TOR_TARGETS = "tor_targets.json" + PSIPHON_CONFIG = "psiphon_config.json" + ANONC_MANIFEST_BUCKET = aws_s3_bucket.anoncred_manifests.bucket + ANONC_MANIFEST_FILE = "manifest.json" + } + secrets = keys(local.ooniapi_deploy_service_secrets.ooniprobe) + service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["ooniprobe"].arn + deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn } module "ooniapi_ooniprobe" { @@ -634,6 +745,10 @@ module "ooniapi_ooniprobe" { module "ooniapi_reverseproxy_deployer" { source = "../../modules/ooniapi_service_deployer" + # Flip to "blue_green" to switch this service to the Docker + systemd + # blue/green deploy on the dedicated Hetzner hosts. + deploy_mode = "ecs" + service_name = "reverseproxy" repo = "ooni/backend" branch_name = "master" @@ -646,6 +761,14 @@ module "ooniapi_reverseproxy_deployer" { ecs_service_name = module.ooniapi_reverseproxy.ecs_service_name ecs_cluster_name = module.ooniapi_cluster.cluster_name + + # Pre-wired for the future flip to deploy_mode = "blue_green" + env_vars = { + TARGET_URL = "https://backend-hel.ooni.org/" + } + secrets = keys(local.ooniapi_deploy_service_secrets.reverseproxy) + service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["reverseproxy"].arn + deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn } module "ooniapi_reverseproxy" { @@ -941,6 +1064,10 @@ module "fastpath_builder" { module "ooniapi_oonirun_deployer" { source = "../../modules/ooniapi_service_deployer" + # Flip to "blue_green" to switch this service to the Docker + systemd + # blue/green deploy on the dedicated Hetzner hosts. + deploy_mode = "ecs" + service_name = "oonirun" repo = "ooni/backend" branch_name = "oonirun-v2-1" @@ -953,6 +1080,12 @@ module "ooniapi_oonirun_deployer" { ecs_service_name = module.ooniapi_oonirun.ecs_service_name ecs_cluster_name = module.ooniapi_cluster.cluster_name + + # Pre-wired for the future flip to deploy_mode = "blue_green" + env_vars = {} + secrets = keys(local.ooniapi_deploy_service_secrets.oonirun) + service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["oonirun"].arn + deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn } module "ooniapi_oonirun" { @@ -992,6 +1125,10 @@ module "ooniapi_oonirun" { module "ooniapi_oonifindings_deployer" { source = "../../modules/ooniapi_service_deployer" + # Flip to "blue_green" to switch this service to the Docker + systemd + # blue/green deploy on the dedicated Hetzner hosts. + deploy_mode = "ecs" + service_name = "oonifindings" repo = "ooni/backend" branch_name = "master" @@ -1004,6 +1141,12 @@ module "ooniapi_oonifindings_deployer" { ecs_service_name = module.ooniapi_oonifindings.ecs_service_name ecs_cluster_name = module.ooniapi_cluster.cluster_name + + # Pre-wired for the future flip to deploy_mode = "blue_green" + env_vars = {} + secrets = keys(local.ooniapi_deploy_service_secrets.oonifindings) + service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["oonifindings"].arn + deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn } module "ooniapi_oonifindings" { @@ -1043,6 +1186,10 @@ module "ooniapi_oonifindings" { module "ooniapi_ooniauth_deployer" { source = "../../modules/ooniapi_service_deployer" + # Flip to "blue_green" to switch this service to the Docker + systemd + # blue/green deploy on the dedicated Hetzner hosts. + deploy_mode = "ecs" + service_name = "ooniauth" repo = "ooni/backend" branch_name = "master" @@ -1055,6 +1202,27 @@ module "ooniapi_ooniauth_deployer" { ecs_service_name = module.ooniapi_ooniauth.ecs_service_name ecs_cluster_name = module.ooniapi_cluster.cluster_name + + # Pre-wired for the future flip to deploy_mode = "blue_green" + env_vars = { + AWS_REGION = var.aws_region + EMAIL_SOURCE_ADDRESS = module.ooniapi_user.email_address + SESSION_EXPIRY_DAYS = 2 + LOGIN_EXPIRY_DAYS = 7 + ADMIN_EMAILS = jsonencode([ + "maja@ooni.org", + "arturo@ooni.org", + "mehul@ooni.org", + "norbel@ooni.org", + "maria@ooni.org", + "admin+dev@ooni.org", + "luis@openobservatory.org", + "contact@openobservatory.org" + ]) + } + secrets = keys(local.ooniapi_deploy_service_secrets.ooniauth) + service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["ooniauth"].arn + deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn } module "ooniapi_ooniauth" { @@ -1112,6 +1280,10 @@ module "ooniapi_ooniauth" { module "ooniapi_oonimeasurements_deployer" { source = "../../modules/ooniapi_service_deployer" + # Flip to "blue_green" to switch this service to the Docker + systemd + # blue/green deploy on the dedicated Hetzner hosts. + deploy_mode = "ecs" + service_name = "oonimeasurements" repo = "ooni/backend" branch_name = "master" @@ -1122,6 +1294,21 @@ module "ooniapi_oonimeasurements_deployer" { codepipeline_bucket = aws_s3_bucket.ooniapi_codepipeline_bucket.bucket + # Pre-wired for the future flip to deploy_mode = "blue_green" + env_vars = { + # it has to be a json-compliant array + OTHER_COLLECTORS = jsonencode([for h in local.fastpath_hosts : "http://${h}:8475"]) + BASE_URL = "https://api.${local.environment}.ooni.io" + S3_BUCKET_NAME = "ooni-data-eu-fra-test" + VALKEY_URL = local.ooniapi_valkey_url + RATE_LIMITS = "10/minute;400000/day;200000/7day" + RATE_LIMITS_WHITELISTED_IPADDRS = jsonencode(["5.9.112.244"]) + RATE_LIMITS_UNMETERED_PAGES = jsonencode(["/metrics", "/health"]) + } + secrets = keys(local.ooniapi_deploy_service_secrets.oonimeasurements) + service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["oonimeasurements"].arn + deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn + ecs_service_name = module.ooniapi_oonimeasurements.ecs_service_name ecs_cluster_name = module.oonitier1plus_cluster.cluster_name } diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf index 2fb42ce7..f3543a25 100644 --- a/tf/environments/prod/main.tf +++ b/tf/environments/prod/main.tf @@ -279,6 +279,96 @@ resource "aws_secretsmanager_secret_version" "oonipg_url" { ) } +### Blue/green deploy secrets (Docker + systemd on dedicated Hetzner hosts) +# +# These mirror the values already passed as `task_secrets` to the ECS task +# definitions above, but consolidated into one JSON blob per service so the +# "blue_green" deploy_mode CodeBuild job can fetch them in a single +# secretsmanager:GetSecretValue call and write each key into the target +# slot's env file. Values are pulled from the same underlying data sources +# used by the ECS `task_secrets` maps, so both deploy paths stay in sync +# automatically. + +data "aws_secretsmanager_secret_version" "ooniapi_user_access_key_id" { + secret_id = module.ooniapi_user.aws_access_key_id_arn +} + +data "aws_secretsmanager_secret_version" "ooniapi_user_secret_access_key" { + secret_id = module.ooniapi_user.aws_secret_access_key_arn +} + +locals { + ooniapi_deploy_service_secrets = { + reverseproxy = { + PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value + } + ooniprobe = { + POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value + JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value + PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_write_url.value + ANONC_SECRET_KEY = data.aws_ssm_parameter.anonc_secret_key.value + } + oonirun = { + POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value + JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value + PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_readonly_url.value + } + oonifindings = { + POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value + JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value + PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value + } + ooniauth = { + POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value + JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value + PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value + ACCOUNT_ID_HASHING_KEY = data.aws_ssm_parameter.account_id_hashing_key.value + AWS_SECRET_ACCESS_KEY = data.aws_secretsmanager_secret_version.ooniapi_user_secret_access_key.secret_string + AWS_ACCESS_KEY_ID = data.aws_secretsmanager_secret_version.ooniapi_user_access_key_id.secret_string + } + oonimeasurements = { + POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value + JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value + PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_oonimeasurements_url.value + ACCOUNT_ID_HASHING_KEY = data.aws_ssm_parameter.account_id_hashing_key.value + } + } +} + +resource "aws_secretsmanager_secret" "ooniapi_deploy_service_secrets" { + for_each = local.ooniapi_deploy_service_secrets + + name = "oonidevops/ooniapi/${each.key}/service_secrets" + tags = local.tags +} + +resource "aws_secretsmanager_secret_version" "ooniapi_deploy_service_secrets" { + for_each = local.ooniapi_deploy_service_secrets + + secret_id = aws_secretsmanager_secret.ooniapi_deploy_service_secrets[each.key].id + secret_string = jsonencode(each.value) +} + +# Shared by every service's blue/green deploy job. The private key itself is +# generated and rotated out-of-band (see the deploy README); Terraform only +# owns the secret container, not its value. +resource "aws_secretsmanager_secret" "ooniapi_deploy_ssh_key" { + name = "oonidevops/ooniapi/deploy_ssh_key" + tags = local.tags +} + +resource "aws_secretsmanager_secret_version" "ooniapi_deploy_ssh_key" { + secret_id = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.id + secret_string = "REPLACE_ME: populate out-of-band with the \"deploy\" user's SSH private key" + + lifecycle { + ignore_changes = [secret_string] + } +} + module "geoip_bucket" { source = "../../modules/s3_bucket" @@ -433,6 +523,10 @@ module "ooni_th_droplet" { module "ooniapi_reverseproxy_deployer" { source = "../../modules/ooniapi_service_deployer" + # Flip to "blue_green" to switch this service to the Docker + systemd + # blue/green deploy on the dedicated Hetzner hosts. + deploy_mode = "ecs" + service_name = "reverseproxy" repo = "ooni/backend" branch_name = "master" @@ -445,6 +539,14 @@ module "ooniapi_reverseproxy_deployer" { ecs_service_name = module.ooniapi_reverseproxy.ecs_service_name ecs_cluster_name = module.ooniapi_cluster.cluster_name + + # Pre-wired for the future flip to deploy_mode = "blue_green" + env_vars = { + TARGET_URL = "https://backend-fsn.ooni.org/" + } + secrets = keys(local.ooniapi_deploy_service_secrets.reverseproxy) + service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["reverseproxy"].arn + deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn } module "ooniapi_reverseproxy" { @@ -857,6 +959,10 @@ EOF module "ooniapi_ooniprobe_deployer" { source = "../../modules/ooniapi_service_deployer" + # Flip to "blue_green" to switch this service to the Docker + systemd + # blue/green deploy on the dedicated Hetzner hosts. + deploy_mode = "ecs" + service_name = "ooniprobe" repo = "ooni/backend" branch_name = "master" @@ -869,6 +975,23 @@ module "ooniapi_ooniprobe_deployer" { ecs_service_name = module.ooniapi_ooniprobe.ecs_service_name ecs_cluster_name = module.ooniapi_cluster.cluster_name + + # Pre-wired for the future flip to deploy_mode = "blue_green" + env_vars = { + # hardcoded IP for fastpath2.prod.prod.ooni.io + FASTPATH_URL = "http://10.0.0.32:8472" + FASTPATH_URLS = jsonencode([for h in local.fastpath_hosts : "http://${h}:8472"]) + FAILED_REPORTS_BUCKET = aws_s3_bucket.ooniprobe_failed_reports.bucket + COLLECTOR_ID = 4 # be sure this is different from dev + CONFIG_BUCKET = aws_s3_bucket.ooni_private_config_bucket.bucket + TOR_TARGETS = "tor_targets.json" + PSIPHON_CONFIG = "psiphon_config.json" + ANONC_MANIFEST_BUCKET = aws_s3_bucket.anoncred_manifests.bucket + ANONC_MANIFEST_FILE = "manifest.json" + } + secrets = keys(local.ooniapi_deploy_service_secrets.ooniprobe) + service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["ooniprobe"].arn + deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn } module "ooniapi_ooniprobe" { @@ -1027,6 +1150,10 @@ module "fastpath_builder" { module "ooniapi_oonirun_deployer" { source = "../../modules/ooniapi_service_deployer" + # Flip to "blue_green" to switch this service to the Docker + systemd + # blue/green deploy on the dedicated Hetzner hosts. + deploy_mode = "ecs" + service_name = "oonirun" repo = "ooni/backend" branch_name = "master" @@ -1039,6 +1166,12 @@ module "ooniapi_oonirun_deployer" { ecs_service_name = module.ooniapi_oonirun.ecs_service_name ecs_cluster_name = module.ooniapi_cluster.cluster_name + + # Pre-wired for the future flip to deploy_mode = "blue_green" + env_vars = {} + secrets = keys(local.ooniapi_deploy_service_secrets.oonirun) + service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["oonirun"].arn + deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn } module "ooniapi_oonirun" { @@ -1079,6 +1212,10 @@ module "ooniapi_oonirun" { module "ooniapi_oonifindings_deployer" { source = "../../modules/ooniapi_service_deployer" + # Flip to "blue_green" to switch this service to the Docker + systemd + # blue/green deploy on the dedicated Hetzner hosts. + deploy_mode = "ecs" + service_name = "oonifindings" repo = "ooni/backend" branch_name = "master" @@ -1091,6 +1228,12 @@ module "ooniapi_oonifindings_deployer" { ecs_service_name = module.ooniapi_oonifindings.ecs_service_name ecs_cluster_name = module.ooniapi_cluster.cluster_name + + # Pre-wired for the future flip to deploy_mode = "blue_green" + env_vars = {} + secrets = keys(local.ooniapi_deploy_service_secrets.oonifindings) + service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["oonifindings"].arn + deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn } module "ooniapi_oonifindings" { @@ -1131,6 +1274,10 @@ module "ooniapi_oonifindings" { module "ooniapi_ooniauth_deployer" { source = "../../modules/ooniapi_service_deployer" + # Flip to "blue_green" to switch this service to the Docker + systemd + # blue/green deploy on the dedicated Hetzner hosts. + deploy_mode = "ecs" + service_name = "ooniauth" repo = "ooni/backend" branch_name = "master" @@ -1143,6 +1290,27 @@ module "ooniapi_ooniauth_deployer" { ecs_service_name = module.ooniapi_ooniauth.ecs_service_name ecs_cluster_name = module.ooniapi_cluster.cluster_name + + # Pre-wired for the future flip to deploy_mode = "blue_green" + env_vars = { + AWS_REGION = var.aws_region + EMAIL_SOURCE_ADDRESS = module.ooniapi_user.email_address + SESSION_EXPIRY_DAYS = 2 + LOGIN_EXPIRY_DAYS = 7 + ADMIN_EMAILS = jsonencode([ + "maja@ooni.org", + "arturo@ooni.org", + "mehul@ooni.org", + "norbel@ooni.org", + "maria@ooni.org", + "admin+dev@ooni.org", + "luis@openobservatory.org", + "contact@openobservatory.org" + ]) + } + secrets = keys(local.ooniapi_deploy_service_secrets.ooniauth) + service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["ooniauth"].arn + deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn } module "ooniapi_ooniauth" { @@ -1202,6 +1370,10 @@ module "ooniapi_ooniauth" { module "ooniapi_oonimeasurements_deployer" { source = "../../modules/ooniapi_service_deployer" + # Flip to "blue_green" to switch this service to the Docker + systemd + # blue/green deploy on the dedicated Hetzner hosts. + deploy_mode = "ecs" + service_name = "oonimeasurements" repo = "ooni/backend" branch_name = "master" @@ -1215,6 +1387,21 @@ module "ooniapi_oonimeasurements_deployer" { ecs_service_name = module.ooniapi_oonimeasurements.ecs_service_name ecs_cluster_name = module.oonitier1plus_cluster.cluster_name # ecs_cluster_name = module.ooniapi_cluster.cluster_name + + # Pre-wired for the future flip to deploy_mode = "blue_green" + env_vars = { + # it has to be a json-compliant array + OTHER_COLLECTORS = jsonencode([for h in local.fastpath_hosts : "http://${h}:8475"]) + BASE_URL = "https://api.ooni.io" + S3_BUCKET_NAME = "ooni-data-eu-fra" + VALKEY_URL = local.ooniapi_valkey_url + RATE_LIMITS = "4000/hour;400000/day;200000/7day" + RATE_LIMITS_WHITELISTED_IPADDRS = jsonencode(["5.9.112.244"]) + RATE_LIMITS_UNMETERED_PAGES = jsonencode(["/metrics", "/health"]) + } + secrets = keys(local.ooniapi_deploy_service_secrets.oonimeasurements) + service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["oonimeasurements"].arn + deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn } module "ooniapi_oonimeasurements" { diff --git a/tf/modules/ooniapi_service_deployer/files/deploy.py b/tf/modules/ooniapi_service_deployer/files/deploy.py new file mode 100644 index 00000000..57cbf5c7 --- /dev/null +++ b/tf/modules/ooniapi_service_deployer/files/deploy.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +# Blue/green deploy for a single OONI API service, run by the +# ooniapi--deploy CodeBuild project (see ../main.tf and +# ./buildspec_deploy.yml). Stdlib only -- no pip installs, so the CodeBuild +# image needs nothing beyond python3, ssh/scp and the aws cli, all of which +# it already ships. +# +# For each of DEPLOY_HOST_PRIMARY/SECONDARY: find the currently idle slot +# (a/b) on that host, write that slot's secrets as individual files (picked +# up by the compose file's file-based `secrets:` entries -- the container +# only ever sees these mounted read-only at /run/secrets/, never as +# environment variables), install & (re)start that slot's docker-compose.yaml +# at the new image tag, health-check it directly on its host port, then flip +# nginx's upstream to it and record the new active slot. +# +# Containers are started with `restart: always`, so the docker daemon +# itself keeps them running across crashes/reboots -- no systemd unit is +# needed on top of docker compose. + +import json +import os +import subprocess +import sys +import tempfile +import time + +SSH_OPTS = [ + "-o", "StrictHostKeyChecking=accept-new", + "-o", "ConnectTimeout=10", + "-i", "/tmp/deploy_key", +] + +# Private, deploy-user-owned (0700) directory files land in before being +# moved/installed into place -- not shared /tmp, so the env file (which +# carries service secrets) is never briefly world-readable on the target +# host. Must match ooniapi_gateway_staging_dir in the Ansible role. +STAGING_DIR = "/var/lib/ooniapi/deploy-staging" + + +def require_env(name): + value = os.environ.get(name) + if not value: + sys.exit(f"missing required environment variable: {name}") + return value + + +def run(cmd, **kwargs): + return subprocess.run(cmd, check=True, text=True, **kwargs) + + +def ssh(user, host, remote_cmd): + return run(["ssh", *SSH_OPTS, f"{user}@{host}", remote_cmd]) + + +def ssh_output(user, host, remote_cmd): + return run(["ssh", *SSH_OPTS, f"{user}@{host}", remote_cmd], capture_output=True).stdout + + +def ssh_succeeds(user, host, remote_cmd): + return subprocess.run(["ssh", *SSH_OPTS, f"{user}@{host}", remote_cmd]).returncode == 0 + + +def scp(user, host, local_path, remote_path): + # -p preserves the local file's mode, so a locally-chmod'd-600 file + # (e.g. the env file) lands on the target host already locked down + # instead of briefly readable at whatever the remote umask would give it. + run(["scp", *SSH_OPTS, "-p", local_path, f"{user}@{host}:{remote_path}"]) + + +def s3_fetch(bucket, key): + path = os.path.join(tempfile.gettempdir(), os.path.basename(key)) + run(["aws", "s3", "cp", f"s3://{bucket}/{key}", path]) + with open(path) as f: + return f.read() + + +def secretsmanager_get(secret_arn): + return run( + ["aws", "secretsmanager", "get-secret-value", "--secret-id", secret_arn, + "--query", "SecretString", "--output", "text"], + capture_output=True, + ).stdout + + +def write_tmp(name, content, mode=0o644): + path = os.path.join(tempfile.gettempdir(), name) + with open(path, "w") as f: + f.write(content) + os.chmod(path, mode) + return path + + +def deploy_host(host, ctx): + service = ctx["service"] + user = ctx["user"] + print(f"=== {service}: deploying to {host} ===") + + active_slot = ssh_output( + user, host, f"cat /etc/ooniapi/{service}/active_slot 2>/dev/null || echo a" + ).strip() or "a" + target_slot = "b" if active_slot == "a" else "a" + target_port = ctx["host_port_b"] if target_slot == "b" else ctx["host_port_a"] + print(f"{service} on {host}: active slot is {active_slot}, deploying to slot {target_slot} (port {target_port})") + + # compose file for the target slot + compose_file = f"{service}-{target_slot}.yaml" + compose_path = f"/etc/ooniapi/{service}/{compose_file}" + compose_content = s3_fetch(ctx["bucket"], f"{service}/{compose_file}") + compose_content = compose_content.replace("__IMAGE_TAG__", ctx["image_tag"]) + compose_local = write_tmp(compose_file, compose_content) + scp(user, host, compose_local, f"{STAGING_DIR}/{compose_file}") + ssh(user, host, f"sudo mv {STAGING_DIR}/{compose_file} {compose_path}") + + # one file per secret, chmod'd 600 locally before scp -p sends it, then + # installed straight to the path the compose file's `secrets:` section + # references (install -D creates the slot's secrets/ dir on first use) + for key, value in ctx["secrets"].items(): + secret_file = f"{service}-{target_slot}-{key}.secret" + secret_local = write_tmp(secret_file, value, mode=0o600) + secret_dest = f"/etc/ooniapi/{service}/{target_slot}/secrets/{key}" + scp(user, host, secret_local, f"{STAGING_DIR}/{secret_file}") + ssh(user, host, + f"sudo install -D -m 600 -o root -g root {STAGING_DIR}/{secret_file} {secret_dest}" + f" && rm -f {STAGING_DIR}/{secret_file}") + + ssh(user, host, f"sudo docker compose -f {compose_path} up -d --pull always --remove-orphans") + + healthy = False + for _ in range(10): + if ssh_succeeds(user, host, f"curl -sf -o /dev/null http://127.0.0.1:{target_port}/health"): + healthy = True + break + time.sleep(2) + if not healthy: + sys.exit(f"{service} on {host}: slot {target_slot} failed health check, aborting deploy") + + state_a, state_b = ("", "down") if target_slot == "a" else ("down", "") + upstream_file = f"{service}-upstream.conf" + upstream_content = s3_fetch(ctx["bucket"], f"{service}/{upstream_file}") + upstream_content = upstream_content.replace("__STATE_A__", state_a).replace("__STATE_B__", state_b) + upstream_local = write_tmp(upstream_file, upstream_content) + scp(user, host, upstream_local, f"{STAGING_DIR}/{upstream_file}") + ssh(user, host, + f"sudo mv {STAGING_DIR}/{upstream_file} /etc/nginx/conf.d/{upstream_file}" + f" && sudo nginx -t && sudo systemctl reload nginx") + ssh(user, host, f"echo {target_slot} | sudo tee /etc/ooniapi/{service}/active_slot > /dev/null") + + print(f"=== {service} on {host}: now serving from slot {target_slot} ===") + + +def main(): + with open("imagedefinitions.json") as f: + image_tag = json.load(f)[0]["imageUri"].rsplit(":", 1)[-1] + + service = require_env("SERVICE_NAME") + print(f"Deploying {service} image tag {image_tag}") + + with open("/tmp/deploy_key", "w") as f: + f.write(secretsmanager_get(require_env("DEPLOY_SSH_KEY_SECRET_ARN"))) + os.chmod("/tmp/deploy_key", 0o600) + + ctx = { + "service": service, + "user": require_env("DEPLOY_SSH_USER"), + "bucket": require_env("DEPLOY_BUCKET"), + "image_tag": image_tag, + "host_port_a": require_env("HOST_PORT_A"), + "host_port_b": require_env("HOST_PORT_B"), + "secrets": json.loads(secretsmanager_get(require_env("SERVICE_SECRETS_ARN"))), + } + + for host in (require_env("DEPLOY_HOST_PRIMARY"), require_env("DEPLOY_HOST_SECONDARY")): + deploy_host(host, ctx) + + +if __name__ == "__main__": + main() diff --git a/tf/modules/ooniapi_service_deployer/main.tf b/tf/modules/ooniapi_service_deployer/main.tf index fb32fefd..46e90700 100755 --- a/tf/modules/ooniapi_service_deployer/main.tf +++ b/tf/modules/ooniapi_service_deployer/main.tf @@ -161,6 +161,211 @@ resource "aws_codebuild_project" "ooniapi" { } } +## Docker Compose blue/green deploy (deploy_mode = "blue_green") + +resource "aws_s3_object" "compose_file" { + for_each = var.deploy_mode == "blue_green" ? { a = var.host_port_a, b = var.host_port_b } : {} + + bucket = var.deploy_bucket + key = "${var.service_name}/${var.service_name}-${each.key}.yaml" + content_type = "text/plain" + + content = templatefile("${path.module}/templates/compose.yaml.tftpl", { + service_name = var.service_name + slot = each.key + host_port = each.value + container_port = var.container_port + network_name = var.network_name + env_vars = var.env_vars + secrets = var.secrets + }) +} + +resource "aws_s3_object" "nginx_upstream" { + count = var.deploy_mode == "blue_green" ? 1 : 0 + + bucket = var.deploy_bucket + key = "${var.service_name}/${var.service_name}-upstream.conf" + content_type = "text/plain" + + content = templatefile("${path.module}/templates/nginx_upstream.conf.tftpl", { + service_name = var.service_name + host_port_a = var.host_port_a + host_port_b = var.host_port_b + }) +} + +resource "aws_s3_object" "deploy_script" { + count = var.deploy_mode == "blue_green" ? 1 : 0 + + bucket = var.deploy_bucket + key = "${var.service_name}/deploy.py" + content_type = "text/x-python" + source = "${path.module}/files/deploy.py" + etag = filemd5("${path.module}/files/deploy.py") +} + +resource "aws_iam_policy" "deploy" { + count = var.deploy_mode == "blue_green" ? 1 : 0 + + description = "Policy used in trust relationship with the blue/green deploy CodeBuild project" + name = "codebuild-deploy-${var.service_name}-${var.aws_region}" + path = "/service-role/" + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents" + ] + Resource = [ + "arn:aws:logs:${var.aws_region}:${local.account_id}:log-group:/aws/codebuild/ooniapi-${var.service_name}-deploy", + "arn:aws:logs:${var.aws_region}:${local.account_id}:log-group:/aws/codebuild/ooniapi-${var.service_name}-deploy:*" + ] + }, + { + # required for CodeBuild to read the CodePipeline BuildArtifact + # (imagedefinitions.json), mirrors the grant on the build role + Effect = "Allow" + Action = [ + "s3:GetObject", + "s3:GetObjectVersion", + "s3:GetBucketAcl", + "s3:GetBucketLocation" + ] + Resource = [ + "arn:aws:s3:::${var.codepipeline_bucket}", + "arn:aws:s3:::${var.codepipeline_bucket}/*" + ] + }, + { + Effect = "Allow" + Action = ["s3:GetObject"] + Resource = ["arn:aws:s3:::${var.deploy_bucket}/${var.service_name}/*"] + }, + { + Effect = "Allow" + Action = ["secretsmanager:GetSecretValue"] + Resource = [ + var.deploy_ssh_key_secret_arn, + var.service_secrets_arn + ] + } + ] + }) +} + +resource "aws_iam_role" "deploy" { + count = var.deploy_mode == "blue_green" ? 1 : 0 + + assume_role_policy = < 0 ~} +secrets: +%{ for name in secrets ~} + ${name}: + file: /etc/ooniapi/${service_name}/${slot}/secrets/${name} +%{ endfor ~} + +%{ endif ~} +services: + ${service_name}-${slot}: + image: docker.io/ooni/api-${service_name}:__IMAGE_TAG__ + container_name: ${service_name}-${slot} + restart: always + ports: + - "127.0.0.1:${host_port}:${container_port}" +%{ if length(env_vars) > 0 ~} + environment: +%{ for key, value in env_vars ~} + ${key}: ${jsonencode(value)} +%{ endfor ~} +%{ endif ~} +%{ if length(secrets) > 0 ~} + secrets: +%{ for name in secrets ~} + - ${name} +%{ endfor ~} +%{ endif ~} diff --git a/tf/modules/ooniapi_service_deployer/templates/nginx_upstream.conf.tftpl b/tf/modules/ooniapi_service_deployer/templates/nginx_upstream.conf.tftpl new file mode 100644 index 00000000..d239d393 --- /dev/null +++ b/tf/modules/ooniapi_service_deployer/templates/nginx_upstream.conf.tftpl @@ -0,0 +1,4 @@ +upstream ${service_name} { + server 127.0.0.1:${host_port_a} __STATE_A__; + server 127.0.0.1:${host_port_b} __STATE_B__; +} diff --git a/tf/modules/ooniapi_service_deployer/variables.tf b/tf/modules/ooniapi_service_deployer/variables.tf index 73e0dc7e..b6056c78 100644 --- a/tf/modules/ooniapi_service_deployer/variables.tf +++ b/tf/modules/ooniapi_service_deployer/variables.tf @@ -30,15 +30,112 @@ variable "trigger_path" { description = "path filter for push changes which trigger the codepipeline eg. ooniapi/services/oonirun/**" } +variable "environment" { + description = "Deployment environment (e.g., prod, dev)" + type = string +} + +variable "deploy_mode" { + description = <<-EOF + Which Deploy stage implementation the pipeline uses: + - "ecs" (default) the existing ECS rolling-deploy stage. + - "blue_green" Docker Compose blue/green deploy to dedicated Hetzner + hosts, driven by a CodeBuild "Deploy" action over SSH. + This is opt-in per service so unmigrated services keep working unchanged. + EOF + type = string + default = "ecs" + + validation { + condition = contains(["ecs", "blue_green"], var.deploy_mode) + error_message = "deploy_mode must be either \"ecs\" or \"blue_green\"." + } +} + +# --- deploy_mode = "ecs" ----------------------------------------------- + variable "ecs_cluster_name" { - description = "id of the cluster to deploy into" + description = "id of the cluster to deploy into. Required when deploy_mode = \"ecs\"." + type = string + default = null } variable "ecs_service_name" { - description = "id of the service in the cluster to deploy" + description = "id of the service in the cluster to deploy. Required when deploy_mode = \"ecs\"." + type = string + default = null } -variable "environment" { - description = "Deployment environment (e.g., prod, dev)" +# --- deploy_mode = "blue_green" ----------------------------------------- + +variable "deploy_bucket" { + description = "S3 bucket that rendered compose files, the nginx upstream conf snippet, and deploy.py are uploaded to. Required when deploy_mode = \"blue_green\"." + type = string + default = null +} + +variable "host_port_a" { + description = "Host port bound to the \"a\" deploy slot. Required when deploy_mode = \"blue_green\"." + type = number + default = null +} + +variable "host_port_b" { + description = "Host port bound to the \"b\" deploy slot. Required when deploy_mode = \"blue_green\"." + type = number + default = null +} + +variable "container_port" { + description = "Port the service listens on inside the container. Required when deploy_mode = \"blue_green\"." + type = number + default = null +} + +variable "network_name" { + description = "Docker network the service's containers attach to. Required when deploy_mode = \"blue_green\"." + type = string + default = null +} + +variable "env_vars" { + description = "Cleartext environment variables for the container. Same shape as ooniapi_service's task_environment (map(string)). Baked directly into the rendered compose file, since none of this is sensitive." + type = map(string) + default = {} +} + +variable "secrets" { + description = "Names of the keys in var.service_secrets_arn's JSON blob. Declared in the rendered compose file as Compose file-based secrets, so each is mounted read-only at /run/secrets/ in the container instead of being exposed as an environment variable. Required when deploy_mode = \"blue_green\"." + type = list(string) + default = [] +} + +variable "service_secrets_arn" { + description = "ARN of the Secrets Manager secret holding the service's runtime secrets as a flat JSON key/value object. Every key becomes a Compose file-based secret written to disk during deploy (see var.secrets). Required when deploy_mode = \"blue_green\"." + type = string + default = null +} + +variable "deploy_ssh_key_secret_arn" { + description = "ARN of the Secrets Manager secret holding the SSH private key the deploy CodeBuild job uses to reach the target hosts. Required when deploy_mode = \"blue_green\"." + type = string + default = null +} + +variable "deploy_host_primary" { + description = "Hostname/IP of the primary dedicated host to deploy to. Required when deploy_mode = \"blue_green\"." + type = string + default = null +} + +variable "deploy_host_secondary" { + description = "Hostname/IP of the secondary dedicated host to deploy to. Required when deploy_mode = \"blue_green\"." + type = string + default = null +} + +variable "deploy_ssh_user" { + description = "SSH user the deploy job connects as on the target hosts." type = string + default = "deploy" }