diff --git a/ansible/deploy-ooniapi-gateway.yml b/ansible/deploy-ooniapi-gateway.yml new file mode 100644 index 00000000..1ff9871c --- /dev/null +++ b/ansible/deploy-ooniapi-gateway.yml @@ -0,0 +1,15 @@ +--- +# backend-hel runs the dev API services (host_vars/backend-hel.ooni.org/ +# ooniapi_gateway.yml). backend-fsn still serves the legacy API and gets the +# gateway only when prod moves. nginx and dehydrated (with ssl_domains from +# host_vars) come first: the gateway role expects both on the host. +- name: Bootstrap the tier0 blue/green gateway on dedicated backend hosts + hosts: backend-hel.ooni.org + become: true + roles: + - role: nginx + tags: nginx + - role: dehydrated + tags: dehydrated + - role: ooniapi_gateway + tags: ooniapi_gateway diff --git a/ansible/host_vars/backend-hel.ooni.org/ooniapi_gateway.yml b/ansible/host_vars/backend-hel.ooni.org/ooniapi_gateway.yml new file mode 100644 index 00000000..227ad4ec --- /dev/null +++ b/ansible/host_vars/backend-hel.ooni.org/ooniapi_gateway.yml @@ -0,0 +1,35 @@ +# backend-hel runs the dev API services, deployed blue/green by their +# deployers in tf/environments/dev (deploy_host_primary), behind the gateway +# vhost of roles/ooniapi_gateway. It answers for the names the dev ALB +# answers for; those keep pointing at the ALB until the DNS cutover. +ooniapi_gateway_primary_domain: api.dev.ooni.io +ooniapi_gateway_direct_domain_suffix: dev.ooni.io + +# testlists stays on its EC2 instance, which only serves plain HTTP: proxy it +# over HTTPS through the dev ALB. testlists.dev.ooni.io must therefore stay on +# the ALB at the cutover. +ooniapi_gateway_testlists_upstream: testlists.dev.ooni.io + +# Must match ooniapi_deploy_ports in tf/environments/dev/main.tf +ooniapi_gateway_services: + - {name: reverseproxy, host_port_a: 18001, host_port_b: 18002} + - {name: ooniprobe, host_port_a: 18011, host_port_b: 18012} + - {name: oonirun, host_port_a: 18021, host_port_b: 18022} + - {name: oonifindings, host_port_a: 18031, host_port_b: 18032} + - {name: ooniauth, host_port_a: 18041, host_port_b: 18042} + - {name: oonimeasurements, host_port_a: 18051, host_port_b: 18052} + +# The certificate for the dev names can only be issued once they resolve +# here. At the cutover, after moving their DNS: add this line to +# ssl_domains, run the playbook with --tags dehydrated, then set +# ooniapi_gateway_cert_ready to true and run it again. +# - "api.dev.ooni.io ooniauth.dev.ooni.io ooniprobe.dev.ooni.io oonirun.dev.ooni.io oonifindings.dev.ooni.io oonimeasurements.dev.ooni.io" +ssl_domains: + - "backend-hel.ooni.org" +ooniapi_gateway_cert_ready: false + +# Public half of dev's oonidevops/ooniapi/deploy_ssh_key secret (eu-central-1, +# dev account), which the deploy jobs log in with. Generate the pair, put the +# private key in the secret and the public key here: +# ssh-keygen -t ed25519 -N '' -C ooniapi-deploy-dev -f deploy_dev +ooniapi_gateway_deploy_ssh_public_key: "" diff --git a/ansible/roles/ooniapi_gateway/defaults/main.yml b/ansible/roles/ooniapi_gateway/defaults/main.yml new file mode 100644 index 00000000..bd607048 --- /dev/null +++ b/ansible/roles/ooniapi_gateway/defaults/main.yml @@ -0,0 +1,111 @@ +--- +# Domain the gateway vhost answers on for path-based routing. Mirrors the +# ALB's `ooniapi_frontend_main_domain_name` in +# tf/modules/ooniapi_frontend/main.tf (e.g. "api.prod.ooni.io"). +# No default: set explicitly per host in host_vars. +ooniapi_gateway_primary_domain: "" + +# Any additional hostnames that should hit the same path-routed vhost. +# Mirrors extra entries in `ooniapi_frontend_alternative_domains` that +# aren't handled elsewhere (e.g. "api.ooni.org" in prod). +ooniapi_gateway_extra_server_names: [] + +# Suffix used for the ALB's secondary, host-header-only routing rules, e.g. +# "oonirun.prod.ooni.io". Mirrors `direct_domain_suffix` in +# tf/modules/ooniapi_frontend/main.tf ("${stage}.ooni.io"). +# No default: set explicitly per host in host_vars. +ooniapi_gateway_direct_domain_suffix: "" + +# Name of the dehydrated cert (the directory under +# ooniapi_gateway_cert_path) that covers ooniapi_gateway_primary_domain, +# every ".{{ ooniapi_gateway_direct_domain_suffix }}" name, and +# ooniapi_gateway_extra_server_names as SANs. +# +# This role does NOT manage that cert -- issuing it means extending (or +# adding to) the `dehydrated` role's ssl_domains on this host, which in turn +# requires DNS for every one of those names to already resolve here. That's +# a DNS-cutover-time step, not a day-1 bootstrap step, so it's deliberately +# left manual. See the role README notes. +ooniapi_gateway_cert_name: "{{ ooniapi_gateway_primary_domain }}" +ooniapi_gateway_cert_path: /var/lib/dehydrated/certs/ + +# Flip to true only once ooniapi_gateway_cert_name actually exists on disk. +# Until then the vhost file is rendered but not installed, so this role is +# safe to run ahead of the real DNS cutover. +ooniapi_gateway_cert_ready: false + +# Services fronted by this gateway, and the two host ports their blue/green +# slots are bound to. host_port_a/host_port_b must match the values passed +# to that service's ooniapi_service_deployer module invocation in Terraform. +# +# Example: +# ooniapi_gateway_services: +# - name: reverseproxy +# host_port_a: 18001 +# host_port_b: 18002 +# - name: oonirun +# host_port_a: 18011 +# host_port_b: 18012 +ooniapi_gateway_services: [] + +# Cross-cloud passthrough for tier0 services not yet migrated off AWS +# (currently just testlists). Set to its public hostname to proxy those +# paths there until testlists itself is migrated; leave empty to omit +# those routes entirely. +ooniapi_gateway_testlists_upstream: "" + +# Docker network shared by every service's containers on this host. +ooniapi_gateway_network_name: ooniapi + +# Valkey for the services' rate limits, reached on the shared network by its +# container name: must match ooniapi_deploy_valkey_url in tf/environments +ooniapi_gateway_valkey_name: valkey +ooniapi_gateway_valkey_image: docker.io/valkey/valkey:8.1-alpine +ooniapi_gateway_valkey_maxmemory: 256mb + +# Routes of the API, shared with the ALB (see the file's header); the +# gateway's locations are generated from them in tasks/main.yml +ooniapi_gateway_routes: "{{ lookup('ansible.builtin.file', role_path ~ '/../../../tf/modules/ooniapi_frontend/routes.yaml') | from_yaml }}" + +# Response caches for the routes marked cache/cache_long in routes.yaml. +# nginx keeps an entry until it goes unused for `inactive`, expired or not, +# so responses valid for minutes and measurements valid for a day get zones +# of their own, and the first can't push the second out. Sized per host, as +# if it got all the traffic (2026-10, data2 query_log and api.ooni.org): +# - long: measurement bodies (raw_measurement, measurement_meta), valid 1 d: +# at most ~50K distinct a day (28K measurement_meta + 25K raw lookups) of +# ~76 KB (max seen ~160 KB), ~3.8 GB a day; 2x that, ~130K keys. +# - short: lists, aggregations, oonirun (valid 10 s to 1 h): ~7.5K requests +# an hour of up to ~60 KB (aggregations 0.3-22 KB, lists 35-70 KB, +# observations ~135 KB), ~450 MB an hour if all were distinct; 4x that. +# 1 MB of keys_zone holds ~8K keys. nginx stores responses uncompressed. +ooniapi_gateway_cache_dir: /var/cache/nginx/ooniapi-gateway +ooniapi_gateway_cache_zones: + short: {keys_zone: 8m, max_size: 2g, inactive: 1h} + long: {keys_zone: 16m, max_size: 8g, inactive: 1d} +# for responses without Cache-Control, like /api/v1/aggregation's today; +# the same as the legacy ooni-api vhost's apicache +ooniapi_gateway_cache_default_valid: 10s +ooniapi_gateway_nginx_user: www-data + +# SSH user the deploy CodeBuild job connects as. Must match +# `deploy_ssh_user` (default "deploy") in the ooniapi_service_deployer +# Terraform module. +ooniapi_gateway_deploy_user: deploy + +# Public half of the key whose private half is in the environment's +# oonidevops/ooniapi/deploy_ssh_key secret, which the deploy job logs in +# with. No default: set per host once that key pair exists. +ooniapi_gateway_deploy_ssh_public_key: "" + +# Private directory the deploy job scp's files into before they're +# moved/installed into place. Must match STAGING_DIR in +# tf/modules/ooniapi_service_deployer/files/deploy.py. +ooniapi_gateway_staging_dir: /var/lib/ooniapi/deploy-staging + +# How long nginx workers left over from a reload may keep serving in-flight +# requests (to the slot a deploy just took out of rotation) before they are +# closed. Matches the ALB's deregistration delay on ECS (AWS default, 300 s), +# and must match drain_timeout in ooniapi_service_deployer: a deploy waits +# this long after the previous flip before it restarts the drained slot. +ooniapi_gateway_drain_timeout: 300 diff --git a/ansible/roles/ooniapi_gateway/filter_plugins/ooniapi_routes.py b/ansible/roles/ooniapi_gateway/filter_plugins/ooniapi_routes.py new file mode 100644 index 00000000..1a17b46d --- /dev/null +++ b/ansible/roles/ooniapi_gateway/filter_plugins/ooniapi_routes.py @@ -0,0 +1,73 @@ +# Filters turning tf/modules/ooniapi_frontend/routes.yaml (shared with the +# ALB) into the gateway's nginx locations; see tasks/main.yml. +# +# nginx picks a location by exact match, then longest prefix, where the ALB +# goes by rule priority. The two agree as long as every ALB pattern is exact +# or a prefix (a trailing *), and no prefix of one service covers a path of +# another; ooniapi_gateway_route_problems lists the routes where they would +# not. + + +def _locations(routes): + out = [] + for r in sorted(routes, key=lambda r: r["priority"]): + # the ALB sends these to ooniprobe_legacy, which the gateway doesn't + # run: they go to ooniprobe, whatever their X-Protocol-Version + if r.get("legacy"): + continue + for p in r.get("paths", []): + prefix = p.endswith("*") + cache = None + if p in r.get("cache_long", []): + cache = "long" + elif r.get("cache"): + cache = "short" + out.append({ + "route": r["name"], + "service": r["service"], + "match": "^~" if prefix else "=", + "path": p[:-1] if prefix else p, + "cache": cache, + }) + return out + + +def ooniapi_gateway_locations(routes): + return _locations(routes) + + +def ooniapi_gateway_direct_hosts(routes): + return list(dict.fromkeys(r["service"] for r in routes if r.get("direct_host"))) + + +def ooniapi_gateway_route_problems(routes): + locations = _locations(routes) + problems = [] + seen = {} + for r in routes: + for p in r.get("cache_long", []): + if p not in r.get("paths", []): + problems.append(f"{r['name']}: cache_long {p} is not one of its paths") + for loc in locations: + if "*" in loc["path"] or "?" in loc["path"]: + problems.append(f"{loc['route']}: {loc['path']}: only exact paths and a trailing * can be served by nginx") + key = (loc["match"], loc["path"]) + if key in seen: + problems.append(f"{loc['route']}: {loc['path']} is also routed by {seen[key]}") + seen.setdefault(key, loc["route"]) + for a in locations: + if a["match"] != "^~": + continue + for b in locations: + if b["service"] != a["service"] and b["path"].startswith(a["path"]): + problems.append(f"{a['route']}: {a['path']}* covers {b['route']}'s {b['path']}, nginx would not follow ALB priorities") + return problems + + +class FilterModule: + def filters(self): + return { + "ooniapi_gateway_locations": ooniapi_gateway_locations, + "ooniapi_gateway_direct_hosts": ooniapi_gateway_direct_hosts, + "ooniapi_gateway_route_problems": ooniapi_gateway_route_problems, + } diff --git a/ansible/roles/ooniapi_gateway/handlers/main.yml b/ansible/roles/ooniapi_gateway/handlers/main.yml new file mode 100644 index 00000000..9d61904d --- /dev/null +++ b/ansible/roles/ooniapi_gateway/handlers/main.yml @@ -0,0 +1,14 @@ +--- +- name: test ooniapi gateway nginx config + ansible.builtin.command: /usr/sbin/nginx -t -c /etc/nginx/nginx.conf + listen: reload ooniapi gateway nginx + +- name: reload ooniapi gateway nginx + ansible.builtin.service: + name: nginx + state: reloaded + +- name: restart docker + ansible.builtin.systemd_service: + name: docker + state: restarted diff --git a/ansible/roles/ooniapi_gateway/tasks/main.yml b/ansible/roles/ooniapi_gateway/tasks/main.yml new file mode 100644 index 00000000..156ba89c --- /dev/null +++ b/ansible/roles/ooniapi_gateway/tasks/main.yml @@ -0,0 +1,295 @@ +--- +# One-time (and idempotent-on-rerun) bootstrap for a host that will run +# tier0 services via Docker Compose blue/green deploys, plus the nginx +# gateway vhost that fronts them. See the Terraform side of this in +# tf/modules/ooniapi_service_deployer (deploy_mode = "blue_green") and its +# files/deploy.py, which is what actually performs each deploy -- this role +# only prepares the host for that job to run against. +# +# Runs as root (become: true at the play level). Each deploy runs `docker +# compose up` as root (via the scoped sudoers rule below), matching this. + +# The deploy job logs in as this user; it may only run what the sudoers rule +# below allows, so it gets no groups of its own +- name: Check the deploy user's public key is set + ansible.builtin.assert: + that: ooniapi_gateway_deploy_ssh_public_key | length > 0 + fail_msg: >- + Set ooniapi_gateway_deploy_ssh_public_key for {{ inventory_hostname }} + to the public half of the oonidevops/ooniapi/deploy_ssh_key secret. + +- name: Create the deploy user + ansible.builtin.user: + name: "{{ ooniapi_gateway_deploy_user }}" + comment: "ooniapi blue/green deploy job" + shell: /bin/bash + create_home: true + +- name: Authorize the deploy job's key + ansible.posix.authorized_key: + user: "{{ ooniapi_gateway_deploy_user }}" + key: "{{ ooniapi_gateway_deploy_ssh_public_key }}" + exclusive: true + +- name: Ensure per-service state directories exist + ansible.builtin.file: + path: "/etc/ooniapi/{{ item.name }}" + state: directory + owner: root + group: root + mode: "0755" + loop: "{{ ooniapi_gateway_services }}" + loop_control: + label: "{{ item.name }}" + +# Files land here (not shared /tmp) before being moved/installed into +# place, so the per-secret files scp'd in on every deploy are never briefly +# world-readable on their way in -- only the deploy user (and root) can +# even list this directory. +- name: Ensure the deploy user's private staging directory exists + ansible.builtin.file: + path: "{{ ooniapi_gateway_staging_dir }}" + state: directory + owner: "{{ ooniapi_gateway_deploy_user }}" + group: "{{ ooniapi_gateway_deploy_user }}" + mode: "0700" + +- name: Check whether each service already has an active_slot marker + ansible.builtin.stat: + path: "/etc/ooniapi/{{ item.name }}/active_slot" + loop: "{{ ooniapi_gateway_services }}" + loop_control: + label: "{{ item.name }}" + register: ooniapi_gateway_active_slot_stat + +- name: Seed active_slot with "a" for services deploying for the first time + ansible.builtin.copy: + dest: "/etc/ooniapi/{{ item.item.name }}/active_slot" + content: "a" + owner: root + group: root + mode: "0644" + loop: "{{ ooniapi_gateway_active_slot_stat.results }}" + loop_control: + label: "{{ item.item.name }}" + when: not item.stat.exists + +- name: Check whether each service already has an nginx upstream conf + ansible.builtin.stat: + path: "/etc/nginx/conf.d/{{ item.name }}-upstream.conf" + loop: "{{ ooniapi_gateway_services }}" + loop_control: + label: "{{ item.name }}" + register: ooniapi_gateway_upstream_stat + +# Seeded with slot "a" up / "b" down, matching active_slot's "a" default +# above, so the very first `nginx -t` (run when installing the gateway +# vhost below) succeeds before that service's first real deploy has run. +- name: Seed a placeholder upstream conf for services deploying for the first time + ansible.builtin.copy: + dest: "/etc/nginx/conf.d/{{ item.item.name }}-upstream.conf" + content: | + map "" $ooniapi_cache_version_{{ item.item.name }} { + default ""; + } + upstream {{ item.item.name }} { + server 127.0.0.1:{{ item.item.host_port_a }}; + server 127.0.0.1:{{ item.item.host_port_b }} down; + } + owner: root + group: root + mode: "0644" + loop: "{{ ooniapi_gateway_upstream_stat.results }}" + loop_control: + label: "{{ item.item.name }}" + when: not item.stat.exists + notify: reload ooniapi gateway nginx + +# Follows https://docs.docker.com/engine/install/debian/ (the officially +# supported docker-ce packages, rather than Debian's own docker.io/ +# docker-compose-v2) so we get upstream's release cadence. The repo URL is +# built from ansible_distribution_release rather than a hardcoded codename +# (e.g. "trixie") so this keeps working unchanged across Debian releases. +- name: Remove conflicting docker packages + ansible.builtin.apt: + name: + - docker.io + - docker-doc + - docker-compose + - docker-compose-v2 + - podman-docker + - containerd + - runc + state: absent + +- name: Install docker's prerequisites + ansible.builtin.apt: + name: + - ca-certificates + - curl + - gnupg + state: present + update_cache: true + +- name: Ensure the apt keyrings directory exists + ansible.builtin.file: + path: /etc/apt/keyrings + state: directory + mode: "0755" + +- name: Download docker's official apt signing key + ansible.builtin.get_url: + url: "https://download.docker.com/linux/{{ ansible_distribution | lower }}/gpg" + dest: /etc/apt/keyrings/docker.asc + mode: "0644" + +- name: Read the fingerprint of the downloaded key + ansible.builtin.command: gpg --with-colons --show-keys /etc/apt/keyrings/docker.asc + register: ooniapi_gateway_docker_key_check + changed_when: false + +# Pins against docker's known-good key fingerprint (9DC8 5822 9FC7 DD38 +# 854A E2D8 8D81 803C 0EBF CD88) so a compromised/MITM'd download of the key +# itself (which apt would otherwise trust blindly via signed-by=) gets +# caught here instead of silently being trusted for every future `apt +# update`. +- name: Verify the downloaded key's fingerprint matches docker's published one + ansible.builtin.assert: + that: + - ooniapi_gateway_docker_key_check.stdout is + search('^fpr:::::::::9DC858229FC7DD38854AE2D88D81803C0EBFCD88:$', multiline=true) + fail_msg: >- + docker.asc's fingerprint does not match the expected + 9DC858229FC7DD38854AE2D88D81803C0EBFCD88 -- refusing to trust it. + success_msg: "docker.asc's fingerprint matches the expected value." + +- name: Get dpkg architecture + ansible.builtin.command: dpkg --print-architecture + register: ooniapi_gateway_dpkg_arch + changed_when: false + +- name: Add the docker apt repository + ansible.builtin.apt_repository: + repo: >- + deb [arch={{ ooniapi_gateway_dpkg_arch.stdout }} signed-by=/etc/apt/keyrings/docker.asc] + https://download.docker.com/linux/{{ ansible_distribution | lower }} + {{ ansible_distribution_release }} stable + filename: docker + state: present + +- name: Install docker + ansible.builtin.apt: + name: + - docker-ce + - docker-ce-cli + - containerd.io + - docker-compose-plugin + state: present + update_cache: true + +- name: Make docker's containerd use the nftables firewall backend + ansible.builtin.lineinfile: + path: /etc/containerd/config.toml + line: 'firewall-backend="nftables"' + state: present + create: true + mode: "0644" + notify: restart docker + +- name: Ensure docker is running + ansible.builtin.systemd_service: + name: docker + state: started + enabled: true + +- name: Check whether the shared docker network already exists + ansible.builtin.command: docker network inspect {{ ooniapi_gateway_network_name }} + register: ooniapi_gateway_network_check + changed_when: false + failed_when: false + +- name: Create the shared docker network + ansible.builtin.command: docker network create {{ ooniapi_gateway_network_name }} + when: ooniapi_gateway_network_check.rc != 0 + +# Rate limit counters of the services on this host (VALKEY_URL +# valkey://:6379), what ElastiCache holds for them on ECS. Reachable +# only on the shared docker network, and kept in memory only: losing the +# counters on a restart resets the limits, as a restart of the services +# would with VALKEY_URL=memory. +- name: Run valkey for the services' rate limits + community.docker.docker_container: + name: "{{ ooniapi_gateway_valkey_name }}" + image: "{{ ooniapi_gateway_valkey_image }}" + state: started + restart_policy: always + networks: + - name: "{{ ooniapi_gateway_network_name }}" + command: >- + valkey-server --save "" --appendonly no + --maxmemory {{ ooniapi_gateway_valkey_maxmemory }} --maxmemory-policy allkeys-lru + +- name: Ensure sudoers.d directory exists + ansible.builtin.file: + path: /etc/sudoers.d + state: directory + owner: root + group: root + +- name: Install scoped sudoers rule for the deploy user + ansible.builtin.template: + src: sudoers-ooniapi-deploy.j2 + dest: /etc/sudoers.d/90-ooniapi-deploy + owner: root + group: root + mode: "0440" + validate: "visudo -cf %s" + +# worker_shutdown_timeout is only valid in nginx's main context, which +# conf.d can't reach; modules-enabled/*.conf is included there by both the +# nginx role's nginx.conf and Debian's stock one. +- name: Bound how long old nginx workers drain after a reload + ansible.builtin.copy: + dest: /etc/nginx/modules-enabled/90-ooniapi-gateway-drain.conf + content: | + # Managed by ansible - roles/ooniapi_gateway (ooniapi_gateway_drain_timeout) + worker_shutdown_timeout {{ ooniapi_gateway_drain_timeout }}s; + owner: root + group: root + mode: "0644" + notify: reload ooniapi gateway nginx + +- name: Derive and check the gateway's locations + ansible.builtin.import_tasks: routes.yml + +- name: Ensure the gateway's cache directories exist + ansible.builtin.file: + path: "{{ ooniapi_gateway_cache_dir }}/{{ item }}" + state: directory + owner: "{{ ooniapi_gateway_nginx_user }}" + group: "{{ ooniapi_gateway_nginx_user }}" + mode: "0700" + loop: "{{ ooniapi_gateway_cache_zones.keys() | list }}" + +- name: Render the gateway vhost + ansible.builtin.template: + src: gateway.conf.j2 + dest: /etc/nginx/conf.d/ooniapi-gateway.conf.pending + owner: root + group: root + mode: "0644" + +# Split into "render" + "install" so a bad template is visible +# (.conf.pending) without ever being loaded by nginx, and so the vhost is +# never installed at all until ooniapi_gateway_cert_ready is true -- see +# defaults/main.yml for why that's a separate, deliberate step. +- name: Install the gateway vhost now that its cert is ready + ansible.builtin.copy: + remote_src: true + src: /etc/nginx/conf.d/ooniapi-gateway.conf.pending + dest: /etc/nginx/conf.d/ooniapi-gateway.conf + owner: root + group: root + mode: "0644" + when: ooniapi_gateway_cert_ready + notify: reload ooniapi gateway nginx diff --git a/ansible/roles/ooniapi_gateway/tasks/routes.yml b/ansible/roles/ooniapi_gateway/tasks/routes.yml new file mode 100644 index 00000000..a6478c7c --- /dev/null +++ b/ansible/roles/ooniapi_gateway/tasks/routes.yml @@ -0,0 +1,14 @@ +--- +# The gateway's locations are generated from routes.yaml, shared with the +# ALB, by filter_plugins/ooniapi_routes.py, which also checks nginx can +# serve them the way the ALB does. +- name: Derive the gateway's locations from routes.yaml + ansible.builtin.set_fact: + ooniapi_gateway_locations: "{{ ooniapi_gateway_routes | ooniapi_gateway_locations }}" + ooniapi_gateway_direct_hosts: "{{ ooniapi_gateway_routes | ooniapi_gateway_direct_hosts }}" + ooniapi_gateway_route_problems: "{{ ooniapi_gateway_routes | ooniapi_gateway_route_problems }}" + +- name: Check that nginx can serve routes.yaml like the ALB does + ansible.builtin.assert: + that: ooniapi_gateway_route_problems | length == 0 + fail_msg: "{{ ooniapi_gateway_route_problems }}" diff --git a/ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 b/ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 new file mode 100644 index 00000000..7a584f24 --- /dev/null +++ b/ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 @@ -0,0 +1,141 @@ +# Managed by ansible - roles/ooniapi_gateway/templates/gateway.conf.j2 +# +# Single "gateway" vhost for the OONI tier0 API on this dedicated host. Its +# locations are generated from tf/modules/ooniapi_frontend/routes.yaml, which +# the AWS ALB's listener rules are generated from too, so both route alike. +# Only routes of services running blue/green here +# ({{ ooniapi_gateway_services | map(attribute='name') | join(', ') }}) and of +# those proxied cross-cloud are handled; everything unmatched falls through +# to "reverseproxy", exactly like the ALB's default_action. +# +# Each service's own `-upstream.conf` (the `upstream { +# ... }` block referenced below via proxy_pass) is dropped into this same +# conf.d directory by that service's blue/green deploy job, not by this +# role -- see ooniapi_service_deployer's buildspec_deploy.yml. A service's +# location block here will fail nginx -t until that service has deployed at +# least once. + +{% set service_names = ooniapi_gateway_services | map(attribute='name') | list %} + +# anonymize ipaddr (same scheme as the legacy ooni-api vhost) +map $remote_addr $ooniapi_gateway_remote_addr_anon { + ~(?P\d+\.\d+\.\d+)\. $ip.0; + ~(?P[^:]+:[^:]+): $ip::; + default 0.0.0.0; +} + +log_format ooniapi_gateway_fmt '$ooniapi_gateway_remote_addr_anon $upstream_cache_status [$time_local] ' + '"$request" $status $body_bytes_sent rt:$request_time "$http_referer" "$http_user_agent"'; + +# response caches, see ooniapi_gateway_cache_zones in defaults/main.yml +{% for name, zone in ooniapi_gateway_cache_zones.items() %} +proxy_cache_path {{ ooniapi_gateway_cache_dir }}/{{ name }} levels=1:2 keys_zone=ooniapi_{{ name }}:{{ zone.keys_zone }} + max_size={{ zone.max_size }} inactive={{ zone.inactive }} use_temp_path=off; +{% endfor %} + +server { + listen 443 ssl http2; + listen [::]:443 ssl http2; + server_name {{ ([ooniapi_gateway_primary_domain] + ooniapi_gateway_extra_server_names) | join(' ') }}; + + access_log syslog:server=unix:/dev/log,tag=ooniapi_gateway,severity=info ooniapi_gateway_fmt; + error_log syslog:server=unix:/dev/log,tag=ooniapi_gateway,severity=info; + + client_max_body_size 200M; # for measurement POST, matches the legacy API vhost + + ssl_certificate {{ ooniapi_gateway_cert_path }}{{ ooniapi_gateway_cert_name }}/fullchain.pem; + ssl_certificate_key {{ ooniapi_gateway_cert_path }}{{ ooniapi_gateway_cert_name }}/privkey.pem; + ssl_trusted_certificate {{ ooniapi_gateway_cert_path }}{{ ooniapi_gateway_cert_name }}/chain.pem; + + # https://ssl-config.mozilla.org/#server=nginx&config=intermediate -- kept + # inline (rather than shared include) so this role has no dependency on + # how nginx itself got installed on this host. + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384; + ssl_prefer_server_ciphers off; + ssl_session_timeout 5m; + ssl_session_cache shared:MozSSL:30m; + ssl_session_tickets off; + ssl_stapling on; + ssl_stapling_verify on; + + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; + add_header X-Frame-Options DENY always; + add_header X-Content-Type-Options nosniff always; + + proxy_http_version 1.1; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 900; + + # caching, for the locations of routes marked cache/cache_long in + # routes.yaml (proxy_cache is off elsewhere). Responses are cached as long + # as their Cache-Control allows (nginx skips no-cache, private, max-age=0 + # and those setting cookies), or ooniapi_gateway_cache_default_valid + # without one. Requests with credentials never use the cache. Each cached + # location keys on its service's $ooniapi_cache_version_, the + # image tag its blue/green deploy writes to -upstream.conf, so a + # deploy invalidates that service's responses, and only those. The key + # includes the Origin header: the services' CORS middleware answers an + # allowed origin with its own Access-Control-Allow-Origin, and a request + # without one (or with another) without CORS headers or Vary, so a + # response cached for one origin must not be served to another. + proxy_cache_methods GET HEAD; + proxy_cache_valid 200 301 302 {{ ooniapi_gateway_cache_default_valid }}; + proxy_cache_valid any 0; + proxy_cache_lock on; + proxy_cache_lock_timeout 30s; + proxy_cache_lock_age 30s; + proxy_cache_use_stale error timeout invalid_header updating; + proxy_cache_bypass $http_authorization; + proxy_no_cache $http_authorization; + add_header X-Cache-Status $upstream_cache_status always; + +{% set remote_upstreams = {'testlists': 'https://' ~ ooniapi_gateway_testlists_upstream} if ooniapi_gateway_testlists_upstream else {} %} + ## --- generated from routes.yaml, see tasks/main.yml --- +{% for l in ooniapi_gateway_locations %} +{% if l.service in service_names %} + location {{ l.match }} {{ l.path }} { proxy_pass http://{{ l.service }};{% if l.cache %} proxy_cache ooniapi_{{ l.cache }}; proxy_cache_key "$ooniapi_cache_version_{{ l.service }}|$http_origin|$scheme$host$request_uri";{% endif %} } # {{ l.route }} +{% elif l.service in remote_upstreams %} + # {{ l.service }} is not yet migrated off AWS, proxied cross-cloud until it is + location {{ l.match }} {{ l.path }} { proxy_pass {{ remote_upstreams[l.service] }}; } # {{ l.route }} +{% endif %} +{% endfor %} + + ## --- default: everything else goes to reverseproxy, same as the ALB's + ## default_action. Covers legacy/unmigrated paths (oonith is NOT among + ## them -- it's routed by host_header on *.th.ooni.org, a hostname this + ## vhost doesn't answer for, so it never reaches here either way). --- + location / { + proxy_pass http://reverseproxy; + } +} + +{% for service in ooniapi_gateway_direct_hosts if service in service_names %} +server { + listen 443 ssl http2; + listen [::]:443 ssl http2; + server_name {{ service }}.{{ ooniapi_gateway_direct_domain_suffix }}; + + access_log syslog:server=unix:/dev/log,tag=ooniapi_gateway,severity=info ooniapi_gateway_fmt; + error_log syslog:server=unix:/dev/log,tag=ooniapi_gateway,severity=info; + + ssl_certificate {{ ooniapi_gateway_cert_path }}{{ ooniapi_gateway_cert_name }}/fullchain.pem; + ssl_certificate_key {{ ooniapi_gateway_cert_path }}{{ ooniapi_gateway_cert_name }}/privkey.pem; + ssl_trusted_certificate {{ ooniapi_gateway_cert_path }}{{ ooniapi_gateway_cert_name }}/chain.pem; + ssl_protocols TLSv1.2 TLSv1.3; + + proxy_http_version 1.1; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 900; + + location / { + proxy_pass http://{{ service }}; + } +} +{% endfor %} diff --git a/ansible/roles/ooniapi_gateway/templates/sudoers-ooniapi-deploy.j2 b/ansible/roles/ooniapi_gateway/templates/sudoers-ooniapi-deploy.j2 new file mode 100644 index 00000000..dacbb55c --- /dev/null +++ b/ansible/roles/ooniapi_gateway/templates/sudoers-ooniapi-deploy.j2 @@ -0,0 +1,29 @@ +# Managed by ansible - roles/ooniapi_gateway/templates/sudoers-ooniapi-deploy.j2 +# +# Scoped to exactly the remote commands issued by +# tf/modules/ooniapi_service_deployer/files/deploy.py over SSH as +# {{ ooniapi_gateway_deploy_user }}. Intentionally narrower than the blanket +# "NOPASSWD:ALL" pattern used for the deploy user in other roles (e.g. +# roles/anonc) -- this user is driven by an automated CI job over the +# network, not a human operator. +# +# Sources below are always files this same job just scp'd into +# {{ ooniapi_gateway_staging_dir }}, a directory only {{ ooniapi_gateway_deploy_user }} +# (and root) can read -- see tasks/main.yml for why that, and not shared +# /tmp, is where the per-secret files (each a Compose file-based secret, +# mounted read-only into its container -- never an environment variable) +# land. +# +# Caveat: sudoers glob matching (fnmatch without FNM_PATHNAME) lets "*" +# match "/", so this is a theoretical rather than practical widening of +# scope, but worth knowing if you're auditing this file. + +Cmnd_Alias OONIAPI_DEPLOY_MV_COMPOSE = /usr/bin/mv {{ ooniapi_gateway_staging_dir }}/*.yaml /etc/ooniapi/*/*.yaml +Cmnd_Alias OONIAPI_DEPLOY_MV_UPSTREAM = /usr/bin/mv {{ ooniapi_gateway_staging_dir }}/*-upstream.conf /etc/nginx/conf.d/*-upstream.conf +Cmnd_Alias OONIAPI_DEPLOY_SECRET_INSTALL = /usr/bin/install -D -m 600 -o root -g root {{ ooniapi_gateway_staging_dir }}/*.secret /etc/ooniapi/*/*/secrets/* +Cmnd_Alias OONIAPI_DEPLOY_COMPOSE_UP = /usr/bin/docker compose -f /etc/ooniapi/*/*.yaml up -d --pull always --remove-orphans +Cmnd_Alias OONIAPI_DEPLOY_NGINX_RELOAD = /usr/bin/systemctl reload nginx +Cmnd_Alias OONIAPI_DEPLOY_NGINX_TEST = /usr/sbin/nginx -t +Cmnd_Alias OONIAPI_DEPLOY_MARKER = /usr/bin/tee /etc/ooniapi/*/active_slot + +{{ ooniapi_gateway_deploy_user }} ALL=(root) NOPASSWD: OONIAPI_DEPLOY_MV_COMPOSE, OONIAPI_DEPLOY_MV_UPSTREAM, OONIAPI_DEPLOY_SECRET_INSTALL, OONIAPI_DEPLOY_COMPOSE_UP, OONIAPI_DEPLOY_NGINX_RELOAD, OONIAPI_DEPLOY_NGINX_TEST, OONIAPI_DEPLOY_MARKER diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 3cdfaf40..f19f259d 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -163,9 +163,7 @@ module "oonipg" { db_allocated_storage = "20" db_max_allocated_storage = null - allow_cidr_blocks = [ - "10.0.0.0/8" - ] + allow_cidr_blocks = concat(["10.0.0.0/8"], local.ooniapi_deploy_host_cidrs) allow_security_groups = [module.ooni_jumphost.ec2_sg_id] tags = merge( @@ -257,6 +255,127 @@ data "aws_ssm_parameter" "account_id_hashing_key" { name = "/oonidevops/secrets/ooni_services/account_id_hashing_key" } +### Blue/green deploy secrets (Docker + systemd on dedicated Hetzner hosts) +# +# These mirror the values already passed as `task_secrets` to the ECS task +# definitions above, but consolidated into one JSON blob per service so the +# "blue_green" deploy_mode CodeBuild job can fetch them in a single +# secretsmanager:GetSecretValue call and write each key into the target +# slot's env file. Values are pulled from the same underlying data sources +# used by the ECS `task_secrets` maps, so both deploy paths stay in sync +# automatically. + +data "aws_secretsmanager_secret_version" "ooniapi_user_access_key_id" { + secret_id = module.ooniapi_user.aws_access_key_id_arn +} + +data "aws_secretsmanager_secret_version" "ooniapi_user_secret_access_key" { + secret_id = module.ooniapi_user.aws_secret_access_key_arn +} + +locals { + ooniapi_deploy_service_secrets = { + reverseproxy = { + PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value + } + ooniprobe = { + POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value + JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret_legacy.value + PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_ooniprobe_url.value + ANONC_SECRET_KEY = data.aws_ssm_parameter.anonc_secret_key.value + } + oonirun = { + POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value + JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value + PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_oonirun_url.value + } + oonifindings = { + POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value + JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value + PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value + } + ooniauth = { + JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value + PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value + ACCOUNT_ID_HASHING_KEY = data.aws_ssm_parameter.account_id_hashing_key.value + AWS_SECRET_ACCESS_KEY = data.aws_secretsmanager_secret_version.ooniapi_user_secret_access_key.secret_string + AWS_ACCESS_KEY_ID = data.aws_secretsmanager_secret_version.ooniapi_user_access_key_id.secret_string + } + oonimeasurements = { + JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value + PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_oonimeasurements_test_url.value + ACCOUNT_ID_HASHING_KEY = data.aws_ssm_parameter.account_id_hashing_key.value + } + } +} + +resource "aws_secretsmanager_secret" "ooniapi_deploy_service_secrets" { + for_each = local.ooniapi_deploy_service_secrets + + name = "oonidevops/ooniapi/${each.key}/service_secrets" + tags = local.tags +} + +resource "aws_secretsmanager_secret_version" "ooniapi_deploy_service_secrets" { + for_each = local.ooniapi_deploy_service_secrets + + secret_id = aws_secretsmanager_secret.ooniapi_deploy_service_secrets[each.key].id + secret_string = jsonencode(each.value) +} + +# Shared by every service's blue/green deploy job. The private key itself is +# generated and rotated out-of-band (see the deploy README); Terraform only +# owns the secret container, not its value. +resource "aws_secretsmanager_secret" "ooniapi_deploy_ssh_key" { + name = "oonidevops/ooniapi/deploy_ssh_key" + tags = local.tags +} + +resource "aws_secretsmanager_secret_version" "ooniapi_deploy_ssh_key" { + secret_id = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.id + secret_string = "REPLACE_ME: populate out-of-band with the \"deploy\" user's SSH private key" + + lifecycle { + ignore_changes = [secret_string] + } +} + +# Where the blue/green deploy job puts the dev services: backend-hel alone. +# The ports must match ooniapi_gateway_services in +# ansible/host_vars/backend-hel.ooni.org/ooniapi_gateway.yml. +data "dns_a_record_set" "ooniapi_deploy_host" { + host = "backend-hel.ooni.org" +} + +locals { + ooniapi_deploy_host = data.dns_a_record_set.ooniapi_deploy_host.host + ooniapi_deploy_network = "ooniapi" + # the valkey container roles/ooniapi_gateway runs on that network + # (ooniapi_gateway_valkey_name), for the services' rate limits + ooniapi_deploy_valkey_url = "valkey://valkey:6379" + # what the services there connect to from: fastpath, Postgres and the + # ClickHouse proxy let it in, as they let in the VPC + ooniapi_deploy_host_cidrs = [for ip in data.dns_a_record_set.ooniapi_deploy_host.addrs : "${ip}/32"] + ooniapi_deploy_ports = { + reverseproxy = [18001, 18002] + ooniprobe = [18011, 18012] + oonirun = [18021, 18022] + oonifindings = [18031, 18032] + ooniauth = [18041, 18042] + oonimeasurements = [18051, 18052] + } +} + +# The compose files, nginx upstream confs and deploy.py the deploy jobs copy +# to the host +resource "aws_s3_bucket" "ooniapi_deploy" { + bucket = "ooniapi-deploy-${var.aws_region}-${random_id.artifact_id.hex}" + tags = local.tags +} + resource "random_id" "artifact_id" { byte_length = 4 } @@ -563,6 +682,11 @@ EOF module "ooniapi_ooniprobe_deployer" { source = "../../modules/ooniapi_service_deployer" + # "blue_green" deploys this service with Docker Compose blue/green to the + # dedicated Hetzner hosts instead of ECS; "both" deploys to ECS, then to + # the hosts, so the ALB stays current until DNS moves. + deploy_mode = "ecs" + service_name = "ooniprobe" repo = "ooni/backend" branch_name = "feat/experiment-versions" @@ -575,6 +699,28 @@ module "ooniapi_ooniprobe_deployer" { ecs_service_name = module.ooniapi_ooniprobe.ecs_service_name ecs_cluster_name = module.ooniapi_cluster.cluster_name + + # Pre-wired for the future flip to deploy_mode = "blue_green" + env_vars = { + FASTPATH_URL = "http://fastpath.${local.environment}.ooni.io:8472" + FASTPATH_URLS = jsonencode(["http://fastpath.${local.environment}.ooni.io:8472"]) # private IPs are not reachable from the host + FAILED_REPORTS_BUCKET = aws_s3_bucket.ooniprobe_failed_reports.bucket + COLLECTOR_ID = 3 # use a different one in prod + CONFIG_BUCKET = aws_s3_bucket.ooni_private_config_bucket.bucket + TOR_TARGETS = "tor_targets.json" + PSIPHON_CONFIG = "psiphon_config.json" + ANONC_MANIFEST_BUCKET = aws_s3_bucket.anoncred_manifests.bucket + ANONC_MANIFEST_FILE = "manifest.json" + } + secrets = keys(local.ooniapi_deploy_service_secrets.ooniprobe) + service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["ooniprobe"].arn + deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn + deploy_bucket = aws_s3_bucket.ooniapi_deploy.bucket + deploy_host_primary = local.ooniapi_deploy_host + network_name = local.ooniapi_deploy_network + container_port = 80 + host_port_a = local.ooniapi_deploy_ports.ooniprobe[0] + host_port_b = local.ooniapi_deploy_ports.ooniprobe[1] } module "ooniapi_ooniprobe" { @@ -693,6 +839,11 @@ module "ooniapi_ooniprobe_legacy" { module "ooniapi_reverseproxy_deployer" { source = "../../modules/ooniapi_service_deployer" + # "blue_green" deploys this service with Docker Compose blue/green to the + # dedicated Hetzner hosts instead of ECS; "both" deploys to ECS, then to + # the hosts, so the ALB stays current until DNS moves. + deploy_mode = "ecs" + service_name = "reverseproxy" repo = "ooni/backend" branch_name = "master" @@ -705,6 +856,20 @@ module "ooniapi_reverseproxy_deployer" { ecs_service_name = module.ooniapi_reverseproxy.ecs_service_name ecs_cluster_name = module.ooniapi_cluster.cluster_name + + # Pre-wired for the future flip to deploy_mode = "blue_green" + env_vars = { + TARGET_URL = "https://backend-hel.ooni.org/" + } + secrets = keys(local.ooniapi_deploy_service_secrets.reverseproxy) + service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["reverseproxy"].arn + deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn + deploy_bucket = aws_s3_bucket.ooniapi_deploy.bucket + deploy_host_primary = local.ooniapi_deploy_host + network_name = local.ooniapi_deploy_network + container_port = 80 + host_port_a = local.ooniapi_deploy_ports.reverseproxy[0] + host_port_b = local.ooniapi_deploy_ports.reverseproxy[1] } module "ooniapi_reverseproxy" { @@ -775,7 +940,8 @@ module "ooni_clickhouse_proxy" { to_port = 9002, // for several clickhouse instances protocol = "tcp", cidr_blocks = concat(module.network.vpc_subnet_private[*].cidr_block, ["${module.ooni_fastpath.aws_instance_private_ip}/32", "${module.ooni_fastpath.aws_instance_public_ip}/32"], - ["${module.ooniapi_testlists.aws_instance_private_ip}/32", "${module.ooniapi_testlists.aws_instance_public_ip}/32"]), + ["${module.ooniapi_testlists.aws_instance_private_ip}/32", "${module.ooniapi_testlists.aws_instance_public_ip}/32"], + local.ooniapi_deploy_host_cidrs), }, { // For the prometheus proxy: from_port = 9200, @@ -974,6 +1140,8 @@ module "ooni_fastpath" { sg_prefix = "oonifastpath" tg_prefix = "fstp" + extra_ingress_cidrs = local.ooniapi_deploy_host_cidrs + monitoring_proxy_private_ip = module.ooni_monitoring_proxy.aws_instance_private_ip monitoring_proxy_public_ip = module.ooni_monitoring_proxy.aws_instance_public_ip @@ -1000,6 +1168,11 @@ module "fastpath_builder" { module "ooniapi_oonirun_deployer" { source = "../../modules/ooniapi_service_deployer" + # "blue_green" deploys this service with Docker Compose blue/green to the + # dedicated Hetzner hosts instead of ECS; "both" deploys to ECS, then to + # the hosts, so the ALB stays current until DNS moves. + deploy_mode = "ecs" + service_name = "oonirun" repo = "ooni/backend" branch_name = "master" @@ -1012,6 +1185,18 @@ module "ooniapi_oonirun_deployer" { ecs_service_name = module.ooniapi_oonirun.ecs_service_name ecs_cluster_name = module.ooniapi_cluster.cluster_name + + # Pre-wired for the future flip to deploy_mode = "blue_green" + env_vars = {} + secrets = keys(local.ooniapi_deploy_service_secrets.oonirun) + service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["oonirun"].arn + deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn + deploy_bucket = aws_s3_bucket.ooniapi_deploy.bucket + deploy_host_primary = local.ooniapi_deploy_host + network_name = local.ooniapi_deploy_network + container_port = 80 + host_port_a = local.ooniapi_deploy_ports.oonirun[0] + host_port_b = local.ooniapi_deploy_ports.oonirun[1] } module "ooniapi_oonirun" { @@ -1051,6 +1236,11 @@ module "ooniapi_oonirun" { module "ooniapi_oonifindings_deployer" { source = "../../modules/ooniapi_service_deployer" + # "blue_green" deploys this service with Docker Compose blue/green to the + # dedicated Hetzner hosts instead of ECS; "both" deploys to ECS, then to + # the hosts, so the ALB stays current until DNS moves. + deploy_mode = "ecs" + service_name = "oonifindings" repo = "ooni/backend" branch_name = "master" @@ -1063,6 +1253,18 @@ module "ooniapi_oonifindings_deployer" { ecs_service_name = module.ooniapi_oonifindings.ecs_service_name ecs_cluster_name = module.ooniapi_cluster.cluster_name + + # Pre-wired for the future flip to deploy_mode = "blue_green" + env_vars = {} + secrets = keys(local.ooniapi_deploy_service_secrets.oonifindings) + service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["oonifindings"].arn + deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn + deploy_bucket = aws_s3_bucket.ooniapi_deploy.bucket + deploy_host_primary = local.ooniapi_deploy_host + network_name = local.ooniapi_deploy_network + container_port = 80 + host_port_a = local.ooniapi_deploy_ports.oonifindings[0] + host_port_b = local.ooniapi_deploy_ports.oonifindings[1] } module "ooniapi_oonifindings" { @@ -1101,6 +1303,11 @@ module "ooniapi_oonifindings" { module "ooniapi_ooniauth_deployer" { source = "../../modules/ooniapi_service_deployer" + # "blue_green" deploys this service with Docker Compose blue/green to the + # dedicated Hetzner hosts instead of ECS; "both" deploys to ECS, then to + # the hosts, so the ALB stays current until DNS moves. + deploy_mode = "ecs" + service_name = "ooniauth" repo = "ooni/backend" branch_name = "master" @@ -1113,6 +1320,33 @@ module "ooniapi_ooniauth_deployer" { ecs_service_name = module.ooniapi_ooniauth.ecs_service_name ecs_cluster_name = module.ooniapi_cluster.cluster_name + + # Pre-wired for the future flip to deploy_mode = "blue_green" + env_vars = { + AWS_REGION = var.aws_region + EMAIL_SOURCE_ADDRESS = module.ooniapi_user.email_address + SESSION_EXPIRY_DAYS = 2 + LOGIN_EXPIRY_DAYS = 7 + ADMIN_EMAILS = jsonencode([ + "maja@ooni.org", + "arturo@ooni.org", + "mehul@ooni.org", + "norbel@ooni.org", + "maria@ooni.org", + "admin+dev@ooni.org", + "luis@openobservatory.org", + "contact@openobservatory.org" + ]) + } + secrets = keys(local.ooniapi_deploy_service_secrets.ooniauth) + service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["ooniauth"].arn + deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn + deploy_bucket = aws_s3_bucket.ooniapi_deploy.bucket + deploy_host_primary = local.ooniapi_deploy_host + network_name = local.ooniapi_deploy_network + container_port = 80 + host_port_a = local.ooniapi_deploy_ports.ooniauth[0] + host_port_b = local.ooniapi_deploy_ports.ooniauth[1] } module "ooniapi_ooniauth" { @@ -1170,6 +1404,11 @@ module "ooniapi_ooniauth" { module "ooniapi_oonimeasurements_deployer" { source = "../../modules/ooniapi_service_deployer" + # "blue_green" deploys this service with Docker Compose blue/green to the + # dedicated Hetzner hosts instead of ECS; "both" deploys to ECS, then to + # the hosts, so the ALB stays current until DNS moves. + deploy_mode = "ecs" + service_name = "oonimeasurements" repo = "ooni/backend" branch_name = "636-better-pagination" @@ -1180,6 +1419,27 @@ module "ooniapi_oonimeasurements_deployer" { codepipeline_bucket = aws_s3_bucket.ooniapi_codepipeline_bucket.bucket + # Pre-wired for the future flip to deploy_mode = "blue_green" + env_vars = { + # it has to be a json-compliant array + OTHER_COLLECTORS = jsonencode(["http://fastpath.${local.environment}.ooni.io:8475"]) # private IPs are not reachable from the host + BASE_URL = "https://api.${local.environment}.ooni.io" + S3_BUCKET_NAME = "ooni-data-eu-fra-test" + VALKEY_URL = local.ooniapi_deploy_valkey_url + RATE_LIMITS = "10/minute;400000/day;200000/7day" + RATE_LIMITS_WHITELISTED_IPADDRS = jsonencode(["5.9.112.244"]) + RATE_LIMITS_UNMETERED_PAGES = jsonencode(["/metrics", "/health"]) + } + secrets = keys(local.ooniapi_deploy_service_secrets.oonimeasurements) + service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["oonimeasurements"].arn + deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn + deploy_bucket = aws_s3_bucket.ooniapi_deploy.bucket + deploy_host_primary = local.ooniapi_deploy_host + network_name = local.ooniapi_deploy_network + container_port = 80 + host_port_a = local.ooniapi_deploy_ports.oonimeasurements[0] + host_port_b = local.ooniapi_deploy_ports.oonimeasurements[1] + ecs_service_name = module.ooniapi_oonimeasurements.ecs_service_name ecs_cluster_name = module.oonitier1plus_cluster.cluster_name } diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf index b8877903..a02d6c7e 100644 --- a/tf/environments/prod/main.tf +++ b/tf/environments/prod/main.tf @@ -287,6 +287,99 @@ resource "aws_secretsmanager_secret_version" "oonipg_url" { ) } +# Left out of prod until a prod service moves to deploy_mode "blue_green" +# or "both": applying them would copy prod's credentials into new Secrets +# Manager secrets that nothing reads yet. Uncomment together with the +# matching block in that service's deployer below. +# +### Blue/green deploy secrets (Docker + systemd on dedicated Hetzner hosts) +# +# These mirror the values already passed as `task_secrets` to the ECS task +# definitions above, but consolidated into one JSON blob per service so the +# "blue_green" deploy_mode CodeBuild job can fetch them in a single +# secretsmanager:GetSecretValue call and write each key into the target +# slot's env file. Values are pulled from the same underlying data sources +# used by the ECS `task_secrets` maps, so both deploy paths stay in sync +# automatically. +# +# data "aws_secretsmanager_secret_version" "ooniapi_user_access_key_id" { +# secret_id = module.ooniapi_user.aws_access_key_id_arn +# } +# +# data "aws_secretsmanager_secret_version" "ooniapi_user_secret_access_key" { +# secret_id = module.ooniapi_user.aws_secret_access_key_arn +# } +# +# locals { +# ooniapi_deploy_service_secrets = { +# reverseproxy = { +# PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value +# } +# ooniprobe = { +# POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value +# JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value +# PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value +# CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_write_url.value +# ANONC_SECRET_KEY = data.aws_ssm_parameter.anonc_secret_key.value +# } +# oonirun = { +# POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value +# JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value +# PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value +# CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_oonirun_url.value +# } +# oonifindings = { +# POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value +# JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value +# PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value +# } +# ooniauth = { +# JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value +# PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value +# ACCOUNT_ID_HASHING_KEY = data.aws_ssm_parameter.account_id_hashing_key.value +# AWS_SECRET_ACCESS_KEY = data.aws_secretsmanager_secret_version.ooniapi_user_secret_access_key.secret_string +# AWS_ACCESS_KEY_ID = data.aws_secretsmanager_secret_version.ooniapi_user_access_key_id.secret_string +# } +# oonimeasurements = { +# JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value +# PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value +# CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_oonimeasurements_url.value +# ACCOUNT_ID_HASHING_KEY = data.aws_ssm_parameter.account_id_hashing_key.value +# } +# } +# } +# +# resource "aws_secretsmanager_secret" "ooniapi_deploy_service_secrets" { +# for_each = local.ooniapi_deploy_service_secrets +# +# name = "oonidevops/ooniapi/${each.key}/service_secrets" +# tags = local.tags +# } +# +# resource "aws_secretsmanager_secret_version" "ooniapi_deploy_service_secrets" { +# for_each = local.ooniapi_deploy_service_secrets +# +# secret_id = aws_secretsmanager_secret.ooniapi_deploy_service_secrets[each.key].id +# secret_string = jsonencode(each.value) +# } +# +# Shared by every service's blue/green deploy job. The private key itself is +# generated and rotated out-of-band (see the deploy README); Terraform only +# owns the secret container, not its value. +# resource "aws_secretsmanager_secret" "ooniapi_deploy_ssh_key" { +# name = "oonidevops/ooniapi/deploy_ssh_key" +# tags = local.tags +# } + +# resource "aws_secretsmanager_secret_version" "ooniapi_deploy_ssh_key" { +# secret_id = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.id +# secret_string = "REPLACE_ME: populate out-of-band with the \"deploy\" user's SSH private key" +# +# lifecycle { +# ignore_changes = [secret_string] +# } +# } + module "geoip_bucket" { source = "../../modules/s3_bucket" @@ -452,6 +545,11 @@ resource "aws_route53_record" "ooni_wc_th" { module "ooniapi_reverseproxy_deployer" { source = "../../modules/ooniapi_service_deployer" + # "blue_green" deploys this service with Docker Compose blue/green to the + # dedicated Hetzner hosts instead of ECS; "both" deploys to ECS, then to + # the hosts, so the ALB stays current until DNS moves. + deploy_mode = "ecs" + service_name = "reverseproxy" repo = "ooni/backend" branch_name = "master" @@ -464,6 +562,15 @@ module "ooniapi_reverseproxy_deployer" { ecs_service_name = module.ooniapi_reverseproxy.ecs_service_name ecs_cluster_name = module.ooniapi_cluster.cluster_name + + # Pre-wired for the future flip to deploy_mode = "blue_green" or "both"; + # commented out with the blue/green deploy secrets above + # env_vars = { + # TARGET_URL = "https://backend-fsn.ooni.org/" + # } + # secrets = keys(local.ooniapi_deploy_service_secrets.reverseproxy) + # service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["reverseproxy"].arn + # deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn } module "ooniapi_reverseproxy" { @@ -876,6 +983,11 @@ EOF module "ooniapi_ooniprobe_deployer" { source = "../../modules/ooniapi_service_deployer" + # "blue_green" deploys this service with Docker Compose blue/green to the + # dedicated Hetzner hosts instead of ECS; "both" deploys to ECS, then to + # the hosts, so the ALB stays current until DNS moves. + deploy_mode = "ecs" + service_name = "ooniprobe" repo = "ooni/backend" branch_name = "master" @@ -888,6 +1000,24 @@ module "ooniapi_ooniprobe_deployer" { ecs_service_name = module.ooniapi_ooniprobe.ecs_service_name ecs_cluster_name = module.ooniapi_cluster.cluster_name + + # Pre-wired for the future flip to deploy_mode = "blue_green" or "both"; + # commented out with the blue/green deploy secrets above + # env_vars = { + # # hardcoded IP for fastpath2.prod.prod.ooni.io + # FASTPATH_URL = "http://10.0.0.32:8472" + # FASTPATH_URLS = jsonencode([for h in local.fastpath_hosts : "http://${h}:8472"]) + # FAILED_REPORTS_BUCKET = aws_s3_bucket.ooniprobe_failed_reports.bucket + # COLLECTOR_ID = 4 # be sure this is different from dev + # CONFIG_BUCKET = aws_s3_bucket.ooni_private_config_bucket.bucket + # TOR_TARGETS = "tor_targets.json" + # PSIPHON_CONFIG = "psiphon_config.json" + # ANONC_MANIFEST_BUCKET = aws_s3_bucket.anoncred_manifests.bucket + # ANONC_MANIFEST_FILE = "manifest.json" + # } + # secrets = keys(local.ooniapi_deploy_service_secrets.ooniprobe) + # service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["ooniprobe"].arn + # deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn } module "ooniapi_ooniprobe" { @@ -1223,6 +1353,11 @@ resource "aws_iam_role_policy" "reuploader_role" { module "ooniapi_oonirun_deployer" { source = "../../modules/ooniapi_service_deployer" + # "blue_green" deploys this service with Docker Compose blue/green to the + # dedicated Hetzner hosts instead of ECS; "both" deploys to ECS, then to + # the hosts, so the ALB stays current until DNS moves. + deploy_mode = "ecs" + service_name = "oonirun" repo = "ooni/backend" branch_name = "master" @@ -1235,6 +1370,13 @@ module "ooniapi_oonirun_deployer" { ecs_service_name = module.ooniapi_oonirun.ecs_service_name ecs_cluster_name = module.ooniapi_cluster.cluster_name + + # Pre-wired for the future flip to deploy_mode = "blue_green" or "both"; + # commented out with the blue/green deploy secrets above + # env_vars = {} + # secrets = keys(local.ooniapi_deploy_service_secrets.oonirun) + # service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["oonirun"].arn + # deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn } module "ooniapi_oonirun" { @@ -1275,6 +1417,11 @@ module "ooniapi_oonirun" { module "ooniapi_oonifindings_deployer" { source = "../../modules/ooniapi_service_deployer" + # "blue_green" deploys this service with Docker Compose blue/green to the + # dedicated Hetzner hosts instead of ECS; "both" deploys to ECS, then to + # the hosts, so the ALB stays current until DNS moves. + deploy_mode = "ecs" + service_name = "oonifindings" repo = "ooni/backend" branch_name = "master" @@ -1287,6 +1434,13 @@ module "ooniapi_oonifindings_deployer" { ecs_service_name = module.ooniapi_oonifindings.ecs_service_name ecs_cluster_name = module.ooniapi_cluster.cluster_name + + # Pre-wired for the future flip to deploy_mode = "blue_green" or "both"; + # commented out with the blue/green deploy secrets above + # env_vars = {} + # secrets = keys(local.ooniapi_deploy_service_secrets.oonifindings) + # service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["oonifindings"].arn + # deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn } module "ooniapi_oonifindings" { @@ -1327,6 +1481,11 @@ module "ooniapi_oonifindings" { module "ooniapi_ooniauth_deployer" { source = "../../modules/ooniapi_service_deployer" + # "blue_green" deploys this service with Docker Compose blue/green to the + # dedicated Hetzner hosts instead of ECS; "both" deploys to ECS, then to + # the hosts, so the ALB stays current until DNS moves. + deploy_mode = "ecs" + service_name = "ooniauth" repo = "ooni/backend" branch_name = "master" @@ -1339,6 +1498,28 @@ module "ooniapi_ooniauth_deployer" { ecs_service_name = module.ooniapi_ooniauth.ecs_service_name ecs_cluster_name = module.ooniapi_cluster.cluster_name + + # Pre-wired for the future flip to deploy_mode = "blue_green" or "both"; + # commented out with the blue/green deploy secrets above + # env_vars = { + # AWS_REGION = var.aws_region + # EMAIL_SOURCE_ADDRESS = module.ooniapi_user.email_address + # SESSION_EXPIRY_DAYS = 2 + # LOGIN_EXPIRY_DAYS = 7 + # ADMIN_EMAILS = jsonencode([ + # "maja@ooni.org", + # "arturo@ooni.org", + # "mehul@ooni.org", + # "norbel@ooni.org", + # "maria@ooni.org", + # "admin+dev@ooni.org", + # "luis@openobservatory.org", + # "contact@openobservatory.org" + # ]) + # } + # secrets = keys(local.ooniapi_deploy_service_secrets.ooniauth) + # service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["ooniauth"].arn + # deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn } module "ooniapi_ooniauth" { @@ -1398,6 +1579,11 @@ module "ooniapi_ooniauth" { module "ooniapi_oonimeasurements_deployer" { source = "../../modules/ooniapi_service_deployer" + # "blue_green" deploys this service with Docker Compose blue/green to the + # dedicated Hetzner hosts instead of ECS; "both" deploys to ECS, then to + # the hosts, so the ALB stays current until DNS moves. + deploy_mode = "ecs" + service_name = "oonimeasurements" repo = "ooni/backend" branch_name = "master" @@ -1411,6 +1597,22 @@ module "ooniapi_oonimeasurements_deployer" { ecs_service_name = module.ooniapi_oonimeasurements.ecs_service_name ecs_cluster_name = module.oonitier1plus_cluster.cluster_name # ecs_cluster_name = module.ooniapi_cluster.cluster_name + + # Pre-wired for the future flip to deploy_mode = "blue_green" or "both"; + # commented out with the blue/green deploy secrets above + # env_vars = { + # # it has to be a json-compliant array + # OTHER_COLLECTORS = jsonencode([for h in local.fastpath_hosts : "http://${h}:8475"]) + # BASE_URL = "https://api.ooni.io" + # S3_BUCKET_NAME = "ooni-data-eu-fra" + # VALKEY_URL = local.ooniapi_valkey_url + # RATE_LIMITS = "4000/hour;400000/day;200000/7day" + # RATE_LIMITS_WHITELISTED_IPADDRS = jsonencode(["5.9.112.244"]) + # RATE_LIMITS_UNMETERED_PAGES = jsonencode(["/metrics", "/health"]) + # } + # secrets = keys(local.ooniapi_deploy_service_secrets.oonimeasurements) + # service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["oonimeasurements"].arn + # deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn } module "ooniapi_oonimeasurements" { diff --git a/tf/modules/ooni_fastpath/main.tf b/tf/modules/ooni_fastpath/main.tf index 379224bd..8831ba11 100644 --- a/tf/modules/ooni_fastpath/main.tf +++ b/tf/modules/ooni_fastpath/main.tf @@ -12,13 +12,13 @@ locals { from_port = 8472 to_port = 8472 protocol = "tcp" - cidr_blocks = local.vpc_cidrs + cidr_blocks = concat(local.vpc_cidrs, var.extra_ingress_cidrs) }, { from_port = 8475 # for serving jsonl files to_port = 8475 protocol = "tcp" - cidr_blocks = local.vpc_cidrs + cidr_blocks = concat(local.vpc_cidrs, var.extra_ingress_cidrs) }, { from_port = 9100 diff --git a/tf/modules/ooni_fastpath/variables.tf b/tf/modules/ooni_fastpath/variables.tf index f53ae3ef..dfdfe39b 100644 --- a/tf/modules/ooni_fastpath/variables.tf +++ b/tf/modules/ooni_fastpath/variables.tf @@ -76,3 +76,9 @@ variable "tags" { type = map(string) default = {} } + +variable "extra_ingress_cidrs" { + description = "Sources outside the VPC that may submit measurements (8472) and read the measurement spool (8475), e.g. the hosts the API services are deployed to blue/green." + type = list(string) + default = [] +} diff --git a/tf/modules/ooniapi_frontend/main.tf b/tf/modules/ooniapi_frontend/main.tf index 2e1bee8e..8b6d0650 100644 --- a/tf/modules/ooniapi_frontend/main.tf +++ b/tf/modules/ooniapi_frontend/main.tf @@ -261,378 +261,151 @@ resource "aws_alb_listener_rule" "ooniapi_th" { tags = var.tags } -resource "aws_lb_listener_rule" "ooniapi_ooniauth_rule" { - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 108 +locals { + # routes.yaml is shared with the gateway on the Hetzner hosts + # (ansible/roles/ooniapi_gateway), so both serve the same routes + routes = yamldecode(file("${path.module}/routes.yaml")) - action { - type = "forward" - target_group_arn = var.ooniapi_ooniauth_target_group_arn + target_groups = { + ooniauth = var.ooniapi_ooniauth_target_group_arn + oonirun = var.ooniapi_oonirun_target_group_arn + ooniprobe = var.ooniapi_ooniprobe_target_group_arn + ooniprobe_legacy = var.ooniapi_ooniprobe_legacy_target_group_arn + oonifindings = var.ooniapi_oonifindings_target_group_arn + oonimeasurements = var.ooniapi_oonimeasurements_target_group_arn + testlists = var.ooniapi_testlists_target_group_arn } - condition { - path_pattern { - values = [ - "/api/v2/ooniauth/*", - "/api/v1/user_register", - "/api/v1/user_login", - "/api/v1/user_refresh_token", - "/api/_/account_metadata", - ] - } - } + listener_rules = { for r in local.routes : r.name => r } } -resource "aws_lb_listener_rule" "ooniapi_ooniauth_rule_host" { - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 109 - - action { - type = "forward" - target_group_arn = var.ooniapi_ooniauth_target_group_arn - } - - condition { - host_header { - values = ["ooniauth.${local.direct_domain_suffix}"] - } - } -} +resource "aws_lb_listener_rule" "route" { + for_each = local.listener_rules -resource "aws_lb_listener_rule" "ooniapi_oonirun_rule" { listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 110 + priority = each.value.priority action { type = "forward" - target_group_arn = var.ooniapi_oonirun_target_group_arn + target_group_arn = local.target_groups[each.value.service] } - condition { - path_pattern { - values = ["/api/v2/oonirun/*"] + dynamic "condition" { + for_each = can(each.value.paths) ? [each.value.paths] : [] + content { + path_pattern { + values = condition.value + } } - } -} -resource "aws_lb_listener_rule" "ooniapi_oonirun_rule_host" { - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 111 - - action { - type = "forward" - target_group_arn = var.ooniapi_oonirun_target_group_arn - } - - condition { - host_header { - values = ["oonirun.${local.direct_domain_suffix}"] + dynamic "condition" { + for_each = try(each.value.direct_host, false) ? [each.value.service] : [] + content { + host_header { + values = ["${condition.value}.${local.direct_domain_suffix}"] + } } } -} - -resource "aws_lb_listener_rule" "ooniapi_ooniprobe_rule" { - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 120 - - action { - type = "forward" - target_group_arn = var.ooniapi_ooniprobe_target_group_arn - } - - condition { - path_pattern { - values = [ - "/api/v2/ooniprobe/*", - "/api/v1/login", - "/api/v1/register", - "/api/v1/update/*", - "/api/v1/check-in*" - ] + dynamic "condition" { + for_each = can(each.value.http_header) ? [each.value.http_header] : [] + content { + http_header { + http_header_name = condition.value.name + values = condition.value.values + } } } } -resource "aws_lb_listener_rule" "ooniapi_ooniprobe_rule_2" { - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 121 - - action { - type = "forward" - target_group_arn = var.ooniapi_ooniprobe_target_group_arn - } - - condition { - path_pattern { - values = [ - "/api/v1/test-helpers*", - "/api/v1/test-list/urls", - "/report*", - "/api/_/show_countries_prioritization", - "/api/_/debug_prioritization" - ] - } - } +# the rules were one resource each before routes.yaml +moved { + from = aws_lb_listener_rule.ooniapi_ooniauth_rule + to = aws_lb_listener_rule.route["ooniauth_rule"] } -resource "aws_lb_listener_rule" "ooniapi_ooniprobe_rule_3_legacy_version" { - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 122 - - action { - type = "forward" - target_group_arn = var.ooniapi_ooniprobe_legacy_target_group_arn - } - - condition { - path_pattern { - values = [ - "/api/v1/manifest*", - "/api/v1/submit_measurement*", - "/api/v1/sign_credential*" - ] - } - } - - condition { - http_header { - http_header_name = "X-Protocol-Version" - values = ["0.1.0"] - } - } +moved { + from = aws_lb_listener_rule.ooniapi_ooniauth_rule_host + to = aws_lb_listener_rule.route["ooniauth_rule_host"] } -resource "aws_lb_listener_rule" "ooniapi_ooniprobe_rule_3_current_version" { - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 124 - - action { - type = "forward" - target_group_arn = var.ooniapi_ooniprobe_target_group_arn - } - - condition { - path_pattern { - values = [ - "/api/v1/manifest*", - "/api/v1/submit_measurement*", - "/api/v1/sign_credential*" - ] - } - } - - # matches any value, but only when the header is present - condition { - http_header { - http_header_name = "X-Protocol-Version" - values = ["*"] - } - } +moved { + from = aws_lb_listener_rule.ooniapi_oonirun_rule + to = aws_lb_listener_rule.route["oonirun_rule"] } -resource "aws_lb_listener_rule" "ooniapi_ooniprobe_rule_3_no_version" { - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 126 - - action { - type = "forward" - target_group_arn = var.ooniapi_ooniprobe_legacy_target_group_arn - } - - # No X-Protocol-Version header at all: rules 122/124 above already - # matched every request that does carry the header, so anything left - # here is header-less and should go to the legacy service. - condition { - path_pattern { - values = [ - "/api/v1/manifest*", - "/api/v1/submit_measurement*", - "/api/v1/sign_credential*" - ] - } - } +moved { + from = aws_lb_listener_rule.ooniapi_oonirun_rule_host + to = aws_lb_listener_rule.route["oonirun_rule_host"] } -resource "aws_lb_listener_rule" "ooniapi_ooniprobe_rule_4" { - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 123 - - action { - type = "forward" - target_group_arn = var.ooniapi_ooniprobe_target_group_arn - } - - condition { - path_pattern { - values = [ - "/bouncer/net-tests*", - "/api/v1/geolookup*", - "/api/v1/collectors*", - "/api/v1/test-list/tor-targets", - "/api/v1/test-list/psiphon-config" - ] - } - } +moved { + from = aws_lb_listener_rule.ooniapi_ooniprobe_rule + to = aws_lb_listener_rule.route["ooniprobe_rule"] } -resource "aws_lb_listener_rule" "ooniapi_ooniprobe_rule_host" { - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 125 - - action { - type = "forward" - target_group_arn = var.ooniapi_ooniprobe_target_group_arn - } - - - condition { - host_header { - values = ["ooniprobe.${local.direct_domain_suffix}"] - } - } - +moved { + from = aws_lb_listener_rule.ooniapi_ooniprobe_rule_2 + to = aws_lb_listener_rule.route["ooniprobe_rule_2"] } -resource "aws_lb_listener_rule" "ooniapi_oonifindings_rule" { - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 130 - - action { - type = "forward" - target_group_arn = var.ooniapi_oonifindings_target_group_arn - } - - condition { - path_pattern { - values = [ - "/api/v1/incidents/*", - ] - } - } +moved { + from = aws_lb_listener_rule.ooniapi_ooniprobe_rule_3_legacy_version + to = aws_lb_listener_rule.route["ooniprobe_rule_3_legacy_version"] } -resource "aws_lb_listener_rule" "ooniapi_oonifindings_rule_host" { - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 131 - - action { - type = "forward" - target_group_arn = var.ooniapi_oonifindings_target_group_arn - } - condition { - host_header { - values = ["oonifindings.${local.direct_domain_suffix}"] - } - } +moved { + from = aws_lb_listener_rule.ooniapi_ooniprobe_rule_4 + to = aws_lb_listener_rule.route["ooniprobe_rule_4"] } -resource "aws_lb_listener_rule" "ooniapi_oonimeasurements_rule_host" { - # hotfix: to allow us to deploy the frontend without the measurements service - count = var.ooniapi_oonimeasurements_target_group_arn != null ? 1 : 0 - - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 139 - - action { - type = "forward" - target_group_arn = var.ooniapi_oonimeasurements_target_group_arn - } - condition { - host_header { - values = ["oonimeasurements.${local.direct_domain_suffix}"] - } - } +moved { + from = aws_lb_listener_rule.ooniapi_ooniprobe_rule_3_current_version + to = aws_lb_listener_rule.route["ooniprobe_rule_3_current_version"] } -resource "aws_lb_listener_rule" "ooniapi_oonimeasurements_rule_1" { - # hotfix: to allow us to deploy the frontend without the measurements service - count = var.ooniapi_oonimeasurements_target_group_arn != null ? 1 : 0 - - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 140 - - action { - type = "forward" - target_group_arn = var.ooniapi_oonimeasurements_target_group_arn - } - - condition { - path_pattern { - values = [ - "/api/v1/measurements/*", - "/api/v1/raw_measurement", - "/api/v1/measurement_meta", - "/api/v1/measurements", - "/api/v1/torsf_stats" - ] - } - } +moved { + from = aws_lb_listener_rule.ooniapi_ooniprobe_rule_host + to = aws_lb_listener_rule.route["ooniprobe_rule_host"] } -resource "aws_lb_listener_rule" "ooniapi_oonimeasurements_rule_2" { - # hotfix: to allow us to deploy the frontend without the measurements service - count = var.ooniapi_oonimeasurements_target_group_arn != null ? 1 : 0 - - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 142 - - action { - type = "forward" - target_group_arn = var.ooniapi_oonimeasurements_target_group_arn - } +moved { + from = aws_lb_listener_rule.ooniapi_ooniprobe_rule_3_no_version + to = aws_lb_listener_rule.route["ooniprobe_rule_3_no_version"] +} - condition { - path_pattern { - values = [ - "/api/v1/aggregation", - "/api/v1/aggregation/*", - "/api/v1/observations", - "/api/v1/analysis", - ] - } - } +moved { + from = aws_lb_listener_rule.ooniapi_oonifindings_rule + to = aws_lb_listener_rule.route["oonifindings_rule"] } -resource "aws_lb_listener_rule" "ooniapi_oonimeasurements_rule_3" { +moved { + from = aws_lb_listener_rule.ooniapi_oonifindings_rule_host + to = aws_lb_listener_rule.route["oonifindings_rule_host"] +} - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 143 +moved { + from = aws_lb_listener_rule.ooniapi_oonimeasurements_rule_host[0] + to = aws_lb_listener_rule.route["oonimeasurements_rule_host"] +} - action { - type = "forward" - target_group_arn = var.ooniapi_oonimeasurements_target_group_arn - } +moved { + from = aws_lb_listener_rule.ooniapi_oonimeasurements_rule_1[0] + to = aws_lb_listener_rule.route["oonimeasurements_rule_1"] +} - condition { - path_pattern { - values = [ - "/api/v1/detector/changepoints", - ] - } - } +moved { + from = aws_lb_listener_rule.ooniapi_oonimeasurements_rule_2[0] + to = aws_lb_listener_rule.route["oonimeasurements_rule_2"] } -resource "aws_lb_listener_rule" "ooniapi_testlists_rule" { - # hotfix: to allow us to deploy the frontend without the testlists service - count = var.ooniapi_testlists_target_group_arn != null ? 1 : 0 - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 144 +moved { + from = aws_lb_listener_rule.ooniapi_oonimeasurements_rule_3 + to = aws_lb_listener_rule.route["oonimeasurements_rule_3"] +} - action { - type = "forward" - target_group_arn = var.ooniapi_testlists_target_group_arn - } - condition { - path_pattern { - values = [ - "/api/_/url-submission/test-list/*", - "/api/_/url-priorities/list", - "/api/_/url-priorities/update", - "/api/v1/url-submission/submit", - "/api/v1/url-submission/update-url", - ] - } - } +moved { + from = aws_lb_listener_rule.ooniapi_testlists_rule[0] + to = aws_lb_listener_rule.route["testlists_rule"] } diff --git a/tf/modules/ooniapi_frontend/routes.yaml b/tf/modules/ooniapi_frontend/routes.yaml new file mode 100644 index 00000000..ea1e430c --- /dev/null +++ b/tf/modules/ooniapi_frontend/routes.yaml @@ -0,0 +1,176 @@ +# Routes of the OONI API, the one place they are declared. Read by: +# - Terraform, tf/modules/ooniapi_frontend: one ALB listener rule per entry +# (aws_lb_listener_rule.route[""]), for the services on ECS. +# - Ansible, roles/ooniapi_gateway: the nginx locations of the gateway on the +# Hetzner hosts, for the services deployed there blue/green. +# Anything no route matches goes to reverseproxy, on both. +# +# Each entry is one ALB rule, so the ALB's limits apply: at most 5 values +# over all its conditions. Fields: +# name resource key of the rule; keep stable, renaming one makes +# Terraform replace the rule +# priority ALB rule priority, lowest matches first. The gateway can't +# order its locations by it, so a prefix of one service must +# not cover a path of another (checked by the gateway role) +# service target service (ALB target group) +# paths ALB path patterns: exact, or ending in * for a prefix +# direct_host route . to the service +# http_header only match requests with this header and one of these values +# (ALB only: they only choose between ooniprobe and its legacy) +# legacy ALB only: ooniprobe_legacy is being retired and does not run +# on the gateway, which sends these paths to ooniprobe +# cache gateway only: cache GET/HEAD responses of these paths in the +# short lived zone (see roles/ooniapi_gateway defaults) +# cache_long gateway only: those of its paths to cache in the long lived zone +# instead, for responses of immutable measurements + +- name: ooniauth_rule + priority: 108 + service: ooniauth + paths: + - "/api/v2/ooniauth/*" + - "/api/v1/user_register" + - "/api/v1/user_login" + - "/api/v1/user_refresh_token" + - "/api/_/account_metadata" + +- name: ooniauth_rule_host + priority: 109 + service: ooniauth + direct_host: true + +- name: oonirun_rule + priority: 110 + service: oonirun + paths: + - "/api/v2/oonirun/*" + cache: true + +- name: oonirun_rule_host + priority: 111 + service: oonirun + direct_host: true + +- name: ooniprobe_rule + priority: 120 + service: ooniprobe + paths: + - "/api/v2/ooniprobe/*" + - "/api/v1/login" + - "/api/v1/register" + - "/api/v1/update/*" + - "/api/v1/check-in*" + +- name: ooniprobe_rule_2 + priority: 121 + service: ooniprobe + paths: + - "/api/v1/test-helpers*" + - "/api/v1/test-list/urls" + - "/report*" + - "/api/_/show_countries_prioritization" + - "/api/_/debug_prioritization" + +- name: ooniprobe_rule_3_legacy_version + priority: 122 + service: ooniprobe_legacy + paths: + - "/api/v1/manifest*" + - "/api/v1/submit_measurement*" + - "/api/v1/sign_credential*" + http_header: + name: X-Protocol-Version + values: ["0.1.0"] + legacy: true + +- name: ooniprobe_rule_4 + priority: 123 + service: ooniprobe + paths: + - "/bouncer/net-tests*" + - "/api/v1/geolookup*" + - "/api/v1/collectors*" + - "/api/v1/test-list/tor-targets" + - "/api/v1/test-list/psiphon-config" + +- name: ooniprobe_rule_3_current_version + priority: 124 + service: ooniprobe + paths: + - "/api/v1/manifest*" + - "/api/v1/submit_measurement*" + - "/api/v1/sign_credential*" + http_header: + name: X-Protocol-Version + values: ["*"] + +- name: ooniprobe_rule_host + priority: 125 + service: ooniprobe + direct_host: true + +- name: ooniprobe_rule_3_no_version + priority: 126 + service: ooniprobe_legacy + paths: + - "/api/v1/manifest*" + - "/api/v1/submit_measurement*" + - "/api/v1/sign_credential*" + legacy: true + +- name: oonifindings_rule + priority: 130 + service: oonifindings + paths: + - "/api/v1/incidents/*" + +- name: oonifindings_rule_host + priority: 131 + service: oonifindings + direct_host: true + +- name: oonimeasurements_rule_host + priority: 139 + service: oonimeasurements + direct_host: true + +- name: oonimeasurements_rule_1 + priority: 140 + service: oonimeasurements + paths: + - "/api/v1/measurements/*" + - "/api/v1/raw_measurement" + - "/api/v1/measurement_meta" + - "/api/v1/measurements" + - "/api/v1/torsf_stats" + cache: true + cache_long: + - "/api/v1/raw_measurement" + - "/api/v1/measurement_meta" + +- name: oonimeasurements_rule_2 + priority: 142 + service: oonimeasurements + paths: + - "/api/v1/aggregation" + - "/api/v1/aggregation/*" + - "/api/v1/observations" + - "/api/v1/analysis" + cache: true + +- name: oonimeasurements_rule_3 + priority: 143 + service: oonimeasurements + paths: + - "/api/v1/detector/changepoints" + cache: true + +- name: testlists_rule + priority: 144 + service: testlists + paths: + - "/api/_/url-submission/test-list/*" + - "/api/_/url-priorities/list" + - "/api/_/url-priorities/update" + - "/api/v1/url-submission/submit" + - "/api/v1/url-submission/update-url" diff --git a/tf/modules/ooniapi_frontend/variables.tf b/tf/modules/ooniapi_frontend/variables.tf index 614056e6..47ff4f10 100644 --- a/tf/modules/ooniapi_frontend/variables.tf +++ b/tf/modules/ooniapi_frontend/variables.tf @@ -38,12 +38,12 @@ variable "ooniapi_oonifindings_target_group_arn" { variable "ooniapi_oonimeasurements_target_group_arn" { description = "arn for the target group of the oonimeasurements service" - default = null + nullable = false } variable "ooniapi_testlists_target_group_arn" { description = "arn for the target group of the testlists service" - default = null + nullable = false } variable "dns_zone_ooni_io" { diff --git a/tf/modules/ooniapi_service_deployer/files/__pycache__/deploy.cpython-314.pyc b/tf/modules/ooniapi_service_deployer/files/__pycache__/deploy.cpython-314.pyc new file mode 100644 index 00000000..6ab989a4 Binary files /dev/null and b/tf/modules/ooniapi_service_deployer/files/__pycache__/deploy.cpython-314.pyc differ diff --git a/tf/modules/ooniapi_service_deployer/files/deploy.py b/tf/modules/ooniapi_service_deployer/files/deploy.py new file mode 100644 index 00000000..8b9d2f6c --- /dev/null +++ b/tf/modules/ooniapi_service_deployer/files/deploy.py @@ -0,0 +1,241 @@ +#!/usr/bin/env python3 +# Blue/green deploy for a single OONI API service, run by the +# ooniapi--deploy CodeBuild project (see ../main.tf and +# ./buildspec_deploy.yml). Stdlib only -- no pip installs, so the CodeBuild +# image needs nothing beyond python3, ssh/scp and the aws cli, all of which +# it already ships. +# +# For DEPLOY_HOST_PRIMARY, then DEPLOY_HOST_SECONDARY if set: find the currently idle slot +# (a/b) on that host, write that slot's secrets as individual files (picked +# up by the compose file's file-based `secrets:` entries -- the container +# only ever sees these mounted read-only at /run/secrets/, never as +# environment variables), install & (re)start that slot's docker-compose.yaml +# at the new image tag, health-check it directly on its host port, then flip +# nginx's upstream to it and record the new active slot. +# +# Containers are started with `restart: always`, so the docker daemon +# itself keeps them running across crashes/reboots -- no systemd unit is +# needed on top of docker compose. + +import json +import logging +import os +import re +import subprocess +import sys +import tempfile +import time + +SSH_OPTS = [ + "-o", "StrictHostKeyChecking=accept-new", + "-o", "ConnectTimeout=10", + "-i", "/tmp/deploy_key", +] + +# Private, deploy-user-owned (0700) directory files land in before being +# moved/installed into place -- not shared /tmp, so the env file (which +# carries service secrets) is never briefly world-readable on the target +# host. Must match ooniapi_gateway_staging_dir in the Ansible role. +STAGING_DIR = "/var/lib/ooniapi/deploy-staging" + +log = logging.getLogger("deploy") + + +def require_env(name): + value = os.environ.get(name) + if not value: + sys.exit(f"missing required environment variable: {name}") + return value + + +def run(cmd, **kwargs): + # every remote and aws command goes through here; secret values are only + # ever written to files, never passed on a command line, so this is safe + log.info("running: %s", " ".join(cmd)) + return subprocess.run(cmd, check=True, text=True, **kwargs) + + +def ssh(user, host, remote_cmd): + return run(["ssh", *SSH_OPTS, f"{user}@{host}", remote_cmd]) + + +def ssh_output(user, host, remote_cmd): + return run(["ssh", *SSH_OPTS, f"{user}@{host}", remote_cmd], capture_output=True).stdout + + +def ssh_succeeds(user, host, remote_cmd): + cmd = ["ssh", *SSH_OPTS, f"{user}@{host}", remote_cmd] + log.info("running: %s", " ".join(cmd)) + return subprocess.run(cmd).returncode == 0 + + +def scp(user, host, local_path, remote_path): + # -p preserves the local file's mode, so a locally-chmod'd-600 file + # (e.g. the env file) lands on the target host already locked down + # instead of briefly readable at whatever the remote umask would give it. + run(["scp", *SSH_OPTS, "-p", local_path, f"{user}@{host}:{remote_path}"]) + + +def s3_fetch(bucket, key): + path = os.path.join(tempfile.gettempdir(), os.path.basename(key)) + run(["aws", "s3", "cp", f"s3://{bucket}/{key}", path]) + with open(path) as f: + return f.read() + + +def secretsmanager_get(secret_arn): + return run( + ["aws", "secretsmanager", "get-secret-value", "--secret-id", secret_arn, + "--query", "SecretString", "--output", "text"], + capture_output=True, + ).stdout + + +def write_tmp(name, content, mode=0o644): + path = os.path.join(tempfile.gettempdir(), name) + with open(path, "w") as f: + f.write(content) + os.chmod(path, mode) + return path + + +def deploy_host(host, ctx): + service = ctx["service"] + user = ctx["user"] + log.info(f"=== {service}: deploying to {host} ===") + + active_slot = ssh_output( + user, host, f"cat /etc/ooniapi/{service}/active_slot 2>/dev/null || echo a" + ).strip() or "a" + + # the slot about to be replaced is the one the previous deploy took out + # of rotation: nginx workers from before that flip may still be serving + # requests to it, for up to drain_timeout (worker_shutdown_timeout in + # the ooniapi_gateway role). Don't restart it under them. + since_flip = int(ssh_output( + user, host, + f"echo $(( $(date +%s) - $(stat -c %Y /etc/ooniapi/{service}/active_slot 2>/dev/null || echo 0) ))", + ).strip()) + wait = ctx["drain_timeout"] - since_flip + if wait > 0: + log.info(f"{service} on {host}: previous deploy flipped {since_flip}s ago, waiting {wait}s for its old slot to drain") + time.sleep(wait) + target_slot = "b" if active_slot == "a" else "a" + target_port = ctx["host_port_b"] if target_slot == "b" else ctx["host_port_a"] + log.info(f"{service} on {host}: active slot is {active_slot}, deploying to slot {target_slot} (port {target_port})") + + # compose file for the target slot + compose_file = f"{service}-{target_slot}.yaml" + compose_path = f"/etc/ooniapi/{service}/{compose_file}" + compose_content = s3_fetch(ctx["bucket"], f"{service}/{compose_file}") + compose_content = compose_content.replace("__IMAGE_TAG__", ctx["image_tag"]) + compose_local = write_tmp(compose_file, compose_content) + scp(user, host, compose_local, f"{STAGING_DIR}/{compose_file}") + ssh(user, host, f"sudo mv {STAGING_DIR}/{compose_file} {compose_path}") + + # one file per secret, chmod'd 600 locally before scp -p sends it, then + # installed straight to the path the compose file's `secrets:` section + # references (install -D creates the slot's secrets/ dir on first use) + for key, value in ctx["secrets"].items(): + secret_file = f"{service}-{target_slot}-{key}.secret" + secret_local = write_tmp(secret_file, value, mode=0o600) + secret_dest = f"/etc/ooniapi/{service}/{target_slot}/secrets/{key}" + scp(user, host, secret_local, f"{STAGING_DIR}/{secret_file}") + ssh(user, host, + f"sudo install -D -m 600 -o root -g root {STAGING_DIR}/{secret_file} {secret_dest}" + f" && rm -f {STAGING_DIR}/{secret_file}") + + ssh(user, host, f"sudo docker compose -f {compose_path} up -d --pull always --remove-orphans") + + # like the ECS target groups' health check (ooniapi_service: timeout 5, + # healthy_threshold 2): each check may take up to 5 s, and the slot must + # pass twice in a row. ECS gives a new task about a minute to come up; + # the deadline defaults to twice that, and is logged so it can be tuned + started = time.monotonic() + passes = 0 + while passes < 2: + if ssh_succeeds(user, host, f"curl -sf --max-time 5 -o /dev/null http://127.0.0.1:{target_port}/health"): + passes += 1 + else: + passes = 0 + elapsed = time.monotonic() - started + if passes < 2 and elapsed > ctx["health_check_timeout"]: + sys.exit(f"{service} on {host}: slot {target_slot} not healthy after {elapsed:.0f}s, aborting deploy") + if passes < 2: + time.sleep(2) + log.info(f"{service} on {host}: slot {target_slot} healthy after {time.monotonic() - started:.0f}s") + + upstream_file = f"{service}-upstream.conf" + upstream_path = f"/etc/nginx/conf.d/{upstream_file}" + state_a, state_b = ("", "down") if target_slot == "a" else ("down", "") + # the image tag doubles as this service's cache version on the gateway + # (see nginx_upstream.conf.tftpl): new tag, new cache keys + upstream_content = ( + s3_fetch(ctx["bucket"], f"{service}/{upstream_file}") + .replace("__STATE_A__", state_a).replace("__STATE_B__", state_b) + .replace("__CACHE_VERSION__", ctx["image_tag"]) + ) + # kept as is, to put back if nginx rejects the new one + previous_content = ssh_output(user, host, f"cat {upstream_path}") + + def install_upstream(content): + scp(user, host, write_tmp(upstream_file, content), f"{STAGING_DIR}/{upstream_file}") + ssh(user, host, f"sudo mv {STAGING_DIR}/{upstream_file} {upstream_path}") + + install_upstream(upstream_content) + if not ssh_succeeds(user, host, "sudo nginx -t"): + # nginx is still running its old config, but a config that fails + # nginx -t must not stay in conf.d: the next reload (another + # service's deploy, a certificate renewal) would fail, and a restart + # would take every service on this host down. Put back the previous + # file, slot and cache version, which nginx -t accepted before. + log.error(f"{service} on {host}: nginx -t failed with slot {target_slot}'s upstream, restoring slot {active_slot}'s") + install_upstream(previous_content) + if not ssh_succeeds(user, host, "sudo nginx -t"): + sys.exit(f"{service} on {host}: nginx -t still fails after restoring slot {active_slot}'s upstream:" + " the nginx config is broken independently of this deploy, fix it before reloading nginx") + sys.exit(f"{service} on {host}: nginx -t failed with slot {target_slot}'s upstream; restored slot" + f" {active_slot}'s, which is still serving. Aborting deploy") + ssh(user, host, "sudo systemctl reload nginx") + ssh(user, host, f"echo {target_slot} | sudo tee /etc/ooniapi/{service}/active_slot > /dev/null") + + log.info(f"=== {service} on {host}: now serving from slot {target_slot} ===") + + +def main(): + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s", stream=sys.stdout) + with open("imagedefinitions.json") as f: + image_tag = json.load(f)[0]["imageUri"].rsplit(":", 1)[-1] + # it ends up in an nginx string and in compose files + if not re.fullmatch(r"[A-Za-z0-9_.-]{1,128}", image_tag): + sys.exit(f"unexpected image tag: {image_tag!r}") + + service = require_env("SERVICE_NAME") + log.info(f"Deploying {service} image tag {image_tag}") + + with open("/tmp/deploy_key", "w") as f: + f.write(secretsmanager_get(require_env("DEPLOY_SSH_KEY_SECRET_ARN"))) + os.chmod("/tmp/deploy_key", 0o600) + + ctx = { + "service": service, + "user": require_env("DEPLOY_SSH_USER"), + "bucket": require_env("DEPLOY_BUCKET"), + "image_tag": image_tag, + "host_port_a": require_env("HOST_PORT_A"), + "host_port_b": require_env("HOST_PORT_B"), + "health_check_timeout": int(os.environ.get("HEALTH_CHECK_TIMEOUT") or 120), + "drain_timeout": int(os.environ.get("DRAIN_TIMEOUT") or 300), + "secrets": json.loads(secretsmanager_get(require_env("SERVICE_SECRETS_ARN"))), + } + + # a single host is enough where downtime while it reboots is acceptable + hosts = [require_env("DEPLOY_HOST_PRIMARY")] + if os.environ.get("DEPLOY_HOST_SECONDARY"): + hosts.append(os.environ["DEPLOY_HOST_SECONDARY"]) + for host in hosts: + deploy_host(host, ctx) + + +if __name__ == "__main__": + main() diff --git a/tf/modules/ooniapi_service_deployer/main.tf b/tf/modules/ooniapi_service_deployer/main.tf index fb32fefd..d6f635b7 100755 --- a/tf/modules/ooniapi_service_deployer/main.tf +++ b/tf/modules/ooniapi_service_deployer/main.tf @@ -5,6 +5,9 @@ data "aws_caller_identity" "current" {} locals { account_id = data.aws_caller_identity.current.account_id env_label = var.environment == "prod" ? "latest" : "dev" + + deploy_ecs = contains(["ecs", "both"], var.deploy_mode) + deploy_blue_green = contains(["blue_green", "both"], var.deploy_mode) } resource "aws_iam_policy" "codebuild" { @@ -161,6 +164,222 @@ resource "aws_codebuild_project" "ooniapi" { } } +## Docker Compose blue/green deploy (deploy_mode = "blue_green" or "both") + +resource "aws_s3_object" "compose_file" { + for_each = local.deploy_blue_green ? { a = var.host_port_a, b = var.host_port_b } : {} + + bucket = var.deploy_bucket + key = "${var.service_name}/${var.service_name}-${each.key}.yaml" + content_type = "text/plain" + + content = templatefile("${path.module}/templates/compose.yaml.tftpl", { + service_name = var.service_name + slot = each.key + host_port = each.value + container_port = var.container_port + network_name = var.network_name + env_vars = var.env_vars + secrets = var.secrets + }) +} + +resource "aws_s3_object" "nginx_upstream" { + count = local.deploy_blue_green ? 1 : 0 + + bucket = var.deploy_bucket + key = "${var.service_name}/${var.service_name}-upstream.conf" + content_type = "text/plain" + + content = templatefile("${path.module}/templates/nginx_upstream.conf.tftpl", { + service_name = var.service_name + host_port_a = var.host_port_a + host_port_b = var.host_port_b + }) +} + +resource "aws_s3_object" "deploy_script" { + count = local.deploy_blue_green ? 1 : 0 + + bucket = var.deploy_bucket + key = "${var.service_name}/deploy.py" + content_type = "text/x-python" + source = "${path.module}/files/deploy.py" + etag = filemd5("${path.module}/files/deploy.py") +} + +resource "aws_iam_policy" "deploy" { + count = local.deploy_blue_green ? 1 : 0 + + description = "Policy used in trust relationship with the blue/green deploy CodeBuild project" + name = "codebuild-deploy-${var.service_name}-${var.aws_region}" + path = "/service-role/" + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents" + ] + Resource = [ + "arn:aws:logs:${var.aws_region}:${local.account_id}:log-group:/aws/codebuild/ooniapi-${var.service_name}-deploy", + "arn:aws:logs:${var.aws_region}:${local.account_id}:log-group:/aws/codebuild/ooniapi-${var.service_name}-deploy:*" + ] + }, + { + # required for CodeBuild to read the CodePipeline BuildArtifact + # (imagedefinitions.json), mirrors the grant on the build role + Effect = "Allow" + Action = [ + "s3:GetObject", + "s3:GetObjectVersion", + "s3:GetBucketAcl", + "s3:GetBucketLocation" + ] + Resource = [ + "arn:aws:s3:::${var.codepipeline_bucket}", + "arn:aws:s3:::${var.codepipeline_bucket}/*" + ] + }, + { + Effect = "Allow" + Action = ["s3:GetObject"] + Resource = ["arn:aws:s3:::${var.deploy_bucket}/${var.service_name}/*"] + }, + { + Effect = "Allow" + Action = ["secretsmanager:GetSecretValue"] + Resource = [ + var.deploy_ssh_key_secret_arn, + var.service_secrets_arn + ] + } + ] + }) +} + +resource "aws_iam_role" "deploy" { + count = local.deploy_blue_green ? 1 : 0 + + assume_role_policy = < 0 ~} +secrets: +%{ for name in secrets ~} + ${name}: + file: /etc/ooniapi/${service_name}/${slot}/secrets/${name} +%{ endfor ~} + +%{ endif ~} +services: + ${service_name}-${slot}: + image: docker.io/ooni/api-${service_name}:__IMAGE_TAG__ + container_name: ${service_name}-${slot} + restart: always + ports: + - "127.0.0.1:${host_port}:${container_port}" +%{ if length(env_vars) > 0 ~} + environment: +%{ for key, value in env_vars ~} + ${key}: ${jsonencode(value)} +%{ endfor ~} +%{ endif ~} +%{ if length(secrets) > 0 ~} + secrets: +%{ for name in secrets ~} + - ${name} +%{ endfor ~} +%{ endif ~} diff --git a/tf/modules/ooniapi_service_deployer/templates/nginx_upstream.conf.tftpl b/tf/modules/ooniapi_service_deployer/templates/nginx_upstream.conf.tftpl new file mode 100644 index 00000000..640eff7f --- /dev/null +++ b/tf/modules/ooniapi_service_deployer/templates/nginx_upstream.conf.tftpl @@ -0,0 +1,12 @@ +# Rendered by deploy.py on every deploy. __CACHE_VERSION__ becomes the +# deployed image tag: it is part of the gateway's cache key for this +# service, so the reload that puts the new slot in rotation also stops +# serving responses cached from the old one. +map "" $ooniapi_cache_version_${service_name} { + default "__CACHE_VERSION__"; +} + +upstream ${service_name} { + server 127.0.0.1:${host_port_a} __STATE_A__; + server 127.0.0.1:${host_port_b} __STATE_B__; +} diff --git a/tf/modules/ooniapi_service_deployer/variables.tf b/tf/modules/ooniapi_service_deployer/variables.tf index 73e0dc7e..59884d5d 100644 --- a/tf/modules/ooniapi_service_deployer/variables.tf +++ b/tf/modules/ooniapi_service_deployer/variables.tf @@ -30,15 +30,128 @@ variable "trigger_path" { description = "path filter for push changes which trigger the codepipeline eg. ooniapi/services/oonirun/**" } +variable "environment" { + description = "Deployment environment (e.g., prod, dev)" + type = string +} + +variable "deploy_mode" { + description = <<-EOF + Which Deploy stage implementation the pipeline uses: + - "ecs" (default) the existing ECS rolling-deploy stage. + - "blue_green" Docker Compose blue/green deploy to dedicated Hetzner + hosts, driven by a CodeBuild "Deploy" action over SSH. + - "both" the ECS deploy, then the blue/green deploy, of the same + image. For the migration: the ALB keeps serving the + current version until DNS moves to the hosts, and can + take traffic back if it has to. + This is opt-in per service so unmigrated services keep working unchanged. + EOF + type = string + default = "ecs" + + validation { + condition = contains(["ecs", "blue_green", "both"], var.deploy_mode) + error_message = "deploy_mode must be \"ecs\", \"blue_green\" or \"both\"." + } +} + +# --- deploy_mode = "ecs" or "both" --------------------------------------- + variable "ecs_cluster_name" { - description = "id of the cluster to deploy into" + description = "id of the cluster to deploy into. Required when deploy_mode is \"ecs\" or \"both\"." + type = string + default = null } variable "ecs_service_name" { - description = "id of the service in the cluster to deploy" + description = "id of the service in the cluster to deploy. Required when deploy_mode is \"ecs\" or \"both\"." + type = string + default = null } -variable "environment" { - description = "Deployment environment (e.g., prod, dev)" +# --- deploy_mode = "blue_green" or "both" -------------------------------- + +variable "deploy_bucket" { + description = "S3 bucket that rendered compose files, the nginx upstream conf snippet, and deploy.py are uploaded to. Required when deploy_mode is \"blue_green\" or \"both\"." + type = string + default = null +} + +variable "host_port_a" { + description = "Host port bound to the \"a\" deploy slot. Required when deploy_mode is \"blue_green\" or \"both\"." + type = number + default = null +} + +variable "host_port_b" { + description = "Host port bound to the \"b\" deploy slot. Required when deploy_mode is \"blue_green\" or \"both\"." + type = number + default = null +} + +variable "health_check_timeout" { + description = "Seconds a new blue/green slot has to pass its /health check twice in a row before the deploy is aborted. The ECS target groups give a new task about 60 s." + type = number + default = 120 +} + +variable "drain_timeout" { + description = "Seconds requests may keep draining from a blue/green slot after it is taken out of rotation; must match ooniapi_gateway_drain_timeout on the hosts. A deploy waits this long after the previous one before restarting that slot. Mirrors the ALB deregistration delay (AWS default, 300 s)." + type = number + default = 300 +} + +variable "container_port" { + description = "Port the service listens on inside the container. Required when deploy_mode is \"blue_green\" or \"both\"." + type = number + default = null +} + +variable "network_name" { + description = "Docker network the service's containers attach to. Required when deploy_mode is \"blue_green\" or \"both\"." + type = string + default = null +} + +variable "env_vars" { + description = "Cleartext environment variables for the container. Same shape as ooniapi_service's task_environment (map(string)). Baked directly into the rendered compose file, since none of this is sensitive." + type = map(string) + default = {} +} + +variable "secrets" { + description = "Names of the keys in var.service_secrets_arn's JSON blob. Declared in the rendered compose file as Compose file-based secrets, so each is mounted read-only at /run/secrets/ in the container instead of being exposed as an environment variable. Required when deploy_mode is \"blue_green\" or \"both\"." + type = list(string) + default = [] +} + +variable "service_secrets_arn" { + description = "ARN of the Secrets Manager secret holding the service's runtime secrets as a flat JSON key/value object. Every key becomes a Compose file-based secret written to disk during deploy (see var.secrets). Required when deploy_mode is \"blue_green\" or \"both\"." + type = string + default = null +} + +variable "deploy_ssh_key_secret_arn" { + description = "ARN of the Secrets Manager secret holding the SSH private key the deploy CodeBuild job uses to reach the target hosts. Required when deploy_mode is \"blue_green\" or \"both\"." + type = string + default = null +} + +variable "deploy_host_primary" { + description = "Hostname/IP of the primary dedicated host to deploy to. Required when deploy_mode is \"blue_green\" or \"both\"." + type = string + default = null +} + +variable "deploy_host_secondary" { + description = "Hostname/IP of the secondary dedicated host to deploy to, after the primary. Optional: without it the service runs on the primary host alone, as dev does." + type = string + default = null +} + +variable "deploy_ssh_user" { + description = "SSH user the deploy job connects as on the target hosts." type = string + default = "deploy" }