Blue green podman: add "both" mode, add nginx cache, improve logging, shared route generation

ansible/deploy-ooniapi-gateway.yml

@@ -0,0 +1,15 @@
+---
+# backend-hel runs the dev API services (host_vars/backend-hel.ooni.org/
+# ooniapi_gateway.yml). backend-fsn still serves the legacy API and gets the
+# gateway only when prod moves. nginx and dehydrated (with ssl_domains from
+# host_vars) come first: the gateway role expects both on the host.
+- name: Bootstrap the tier0 blue/green gateway on dedicated backend hosts
+  hosts: backend-hel.ooni.org
+  become: true
+  roles:
+    - role: nginx
+      tags: nginx
+    - role: dehydrated
+      tags: dehydrated
+    - role: ooniapi_gateway
+      tags: ooniapi_gateway

ansible/host_vars/backend-hel.ooni.org/ooniapi_gateway.yml

@@ -0,0 +1,35 @@
+# backend-hel runs the dev API services, deployed blue/green by their
+# deployers in tf/environments/dev (deploy_host_primary), behind the gateway
+# vhost of roles/ooniapi_gateway. It answers for the names the dev ALB
+# answers for; those keep pointing at the ALB until the DNS cutover.
+ooniapi_gateway_primary_domain: api.dev.ooni.io
+ooniapi_gateway_direct_domain_suffix: dev.ooni.io
+
+# testlists stays on its EC2 instance, which only serves plain HTTP: proxy it
+# over HTTPS through the dev ALB. testlists.dev.ooni.io must therefore stay on
+# the ALB at the cutover.
+ooniapi_gateway_testlists_upstream: testlists.dev.ooni.io
+
+# Must match ooniapi_deploy_ports in tf/environments/dev/main.tf
+ooniapi_gateway_services:
+  - {name: reverseproxy, host_port_a: 18001, host_port_b: 18002}
+  - {name: ooniprobe, host_port_a: 18011, host_port_b: 18012}
+  - {name: oonirun, host_port_a: 18021, host_port_b: 18022}
+  - {name: oonifindings, host_port_a: 18031, host_port_b: 18032}
+  - {name: ooniauth, host_port_a: 18041, host_port_b: 18042}
+  - {name: oonimeasurements, host_port_a: 18051, host_port_b: 18052}
+
+# The certificate for the dev names can only be issued once they resolve
+# here. At the cutover, after moving their DNS: add this line to
+# ssl_domains, run the playbook with --tags dehydrated, then set
+# ooniapi_gateway_cert_ready to true and run it again.
+#   - "api.dev.ooni.io ooniauth.dev.ooni.io ooniprobe.dev.ooni.io oonirun.dev.ooni.io oonifindings.dev.ooni.io oonimeasurements.dev.ooni.io"
+ssl_domains:
+  - "backend-hel.ooni.org"
+ooniapi_gateway_cert_ready: false
+
+# Public half of dev's oonidevops/ooniapi/deploy_ssh_key secret (eu-central-1,
+# dev account), which the deploy jobs log in with. Generate the pair, put the
+# private key in the secret and the public key here:
+#   ssh-keygen -t ed25519 -N '' -C ooniapi-deploy-dev -f deploy_dev
+ooniapi_gateway_deploy_ssh_public_key: ""

ansible/roles/ooniapi_gateway/defaults/main.yml

@@ -0,0 +1,111 @@
+---
+# Domain the gateway vhost answers on for path-based routing. Mirrors the
+# ALB's `ooniapi_frontend_main_domain_name` in
+# tf/modules/ooniapi_frontend/main.tf (e.g. "api.prod.ooni.io").
+# No default: set explicitly per host in host_vars.
+ooniapi_gateway_primary_domain: ""
+
+# Any additional hostnames that should hit the same path-routed vhost.
+# Mirrors extra entries in `ooniapi_frontend_alternative_domains` that
+# aren't handled elsewhere (e.g. "api.ooni.org" in prod).
+ooniapi_gateway_extra_server_names: []
+
+# Suffix used for the ALB's secondary, host-header-only routing rules, e.g.
+# "oonirun.prod.ooni.io". Mirrors `direct_domain_suffix` in
+# tf/modules/ooniapi_frontend/main.tf ("${stage}.ooni.io").
+# No default: set explicitly per host in host_vars.
+ooniapi_gateway_direct_domain_suffix: ""
+
+# Name of the dehydrated cert (the directory under
+# ooniapi_gateway_cert_path) that covers ooniapi_gateway_primary_domain,
+# every "<service>.{{ ooniapi_gateway_direct_domain_suffix }}" name, and
+# ooniapi_gateway_extra_server_names as SANs.
+#
+# This role does NOT manage that cert -- issuing it means extending (or
+# adding to) the `dehydrated` role's ssl_domains on this host, which in turn
+# requires DNS for every one of those names to already resolve here. That's
+# a DNS-cutover-time step, not a day-1 bootstrap step, so it's deliberately
+# left manual. See the role README notes.
+ooniapi_gateway_cert_name: "{{ ooniapi_gateway_primary_domain }}"
+ooniapi_gateway_cert_path: /var/lib/dehydrated/certs/
+
+# Flip to true only once ooniapi_gateway_cert_name actually exists on disk.
+# Until then the vhost file is rendered but not installed, so this role is
+# safe to run ahead of the real DNS cutover.
+ooniapi_gateway_cert_ready: false
+
+# Services fronted by this gateway, and the two host ports their blue/green
+# slots are bound to. host_port_a/host_port_b must match the values passed
+# to that service's ooniapi_service_deployer module invocation in Terraform.
+#
+# Example:
+# ooniapi_gateway_services:
+#   - name: reverseproxy
+#     host_port_a: 18001
+#     host_port_b: 18002
+#   - name: oonirun
+#     host_port_a: 18011
+#     host_port_b: 18012
+ooniapi_gateway_services: []
+
+# Cross-cloud passthrough for tier0 services not yet migrated off AWS
+# (currently just testlists). Set to its public hostname to proxy those
+# paths there until testlists itself is migrated; leave empty to omit
+# those routes entirely.
+ooniapi_gateway_testlists_upstream: ""
+
+# Docker network shared by every service's containers on this host.
+ooniapi_gateway_network_name: ooniapi
+
+# Valkey for the services' rate limits, reached on the shared network by its
+# container name: must match ooniapi_deploy_valkey_url in tf/environments
+ooniapi_gateway_valkey_name: valkey
+ooniapi_gateway_valkey_image: docker.io/valkey/valkey:8.1-alpine
+ooniapi_gateway_valkey_maxmemory: 256mb
+
+# Routes of the API, shared with the ALB (see the file's header); the
+# gateway's locations are generated from them in tasks/main.yml
+ooniapi_gateway_routes: "{{ lookup('ansible.builtin.file', role_path ~ '/../../../tf/modules/ooniapi_frontend/routes.yaml') | from_yaml }}"
+
+# Response caches for the routes marked cache/cache_long in routes.yaml.
+# nginx keeps an entry until it goes unused for `inactive`, expired or not,
+# so responses valid for minutes and measurements valid for a day get zones
+# of their own, and the first can't push the second out. Sized per host, as
+# if it got all the traffic (2026-10, data2 query_log and api.ooni.org):
+# - long: measurement bodies (raw_measurement, measurement_meta), valid 1 d:
+#   at most ~50K distinct a day (28K measurement_meta + 25K raw lookups) of
+#   ~76 KB (max seen ~160 KB), ~3.8 GB a day; 2x that, ~130K keys.
+# - short: lists, aggregations, oonirun (valid 10 s to 1 h): ~7.5K requests
+#   an hour of up to ~60 KB (aggregations 0.3-22 KB, lists 35-70 KB,
+#   observations ~135 KB), ~450 MB an hour if all were distinct; 4x that.
+# 1 MB of keys_zone holds ~8K keys. nginx stores responses uncompressed.
+ooniapi_gateway_cache_dir: /var/cache/nginx/ooniapi-gateway
+ooniapi_gateway_cache_zones:
+  short: {keys_zone: 8m, max_size: 2g, inactive: 1h}
+  long: {keys_zone: 16m, max_size: 8g, inactive: 1d}
+# for responses without Cache-Control, like /api/v1/aggregation's today;
+# the same as the legacy ooni-api vhost's apicache
+ooniapi_gateway_cache_default_valid: 10s
+ooniapi_gateway_nginx_user: www-data
+
+# SSH user the deploy CodeBuild job connects as. Must match
+# `deploy_ssh_user` (default "deploy") in the ooniapi_service_deployer
+# Terraform module.
+ooniapi_gateway_deploy_user: deploy
+
+# Public half of the key whose private half is in the environment's
+# oonidevops/ooniapi/deploy_ssh_key secret, which the deploy job logs in
+# with. No default: set per host once that key pair exists.
+ooniapi_gateway_deploy_ssh_public_key: ""
+
+# Private directory the deploy job scp's files into before they're
+# moved/installed into place. Must match STAGING_DIR in
+# tf/modules/ooniapi_service_deployer/files/deploy.py.
+ooniapi_gateway_staging_dir: /var/lib/ooniapi/deploy-staging
+
+# How long nginx workers left over from a reload may keep serving in-flight
+# requests (to the slot a deploy just took out of rotation) before they are
+# closed. Matches the ALB's deregistration delay on ECS (AWS default, 300 s),
+# and must match drain_timeout in ooniapi_service_deployer: a deploy waits
+# this long after the previous flip before it restarts the drained slot.
+ooniapi_gateway_drain_timeout: 300

ansible/roles/ooniapi_gateway/filter_plugins/ooniapi_routes.py

@@ -0,0 +1,73 @@
+# Filters turning tf/modules/ooniapi_frontend/routes.yaml (shared with the
+# ALB) into the gateway's nginx locations; see tasks/main.yml.
+#
+# nginx picks a location by exact match, then longest prefix, where the ALB
+# goes by rule priority. The two agree as long as every ALB pattern is exact
+# or a prefix (a trailing *), and no prefix of one service covers a path of
+# another; ooniapi_gateway_route_problems lists the routes where they would
+# not.
+
+
+def _locations(routes):
+    out = []
+    for r in sorted(routes, key=lambda r: r["priority"]):
+        # the ALB sends these to ooniprobe_legacy, which the gateway doesn't
+        # run: they go to ooniprobe, whatever their X-Protocol-Version
+        if r.get("legacy"):
+            continue
+        for p in r.get("paths", []):
+            prefix = p.endswith("*")
+            cache = None
+            if p in r.get("cache_long", []):
+                cache = "long"
+            elif r.get("cache"):
+                cache = "short"
+            out.append({
+                "route": r["name"],
+                "service": r["service"],
+                "match": "^~" if prefix else "=",
+                "path": p[:-1] if prefix else p,
+                "cache": cache,
+            })
+    return out
+
+
+def ooniapi_gateway_locations(routes):
+    return _locations(routes)
+
+
+def ooniapi_gateway_direct_hosts(routes):
+    return list(dict.fromkeys(r["service"] for r in routes if r.get("direct_host")))
+
+
+def ooniapi_gateway_route_problems(routes):
+    locations = _locations(routes)
+    problems = []
+    seen = {}
+    for r in routes:
+        for p in r.get("cache_long", []):
+            if p not in r.get("paths", []):
+                problems.append(f"{r['name']}: cache_long {p} is not one of its paths")
+    for loc in locations:
+        if "*" in loc["path"] or "?" in loc["path"]:
+            problems.append(f"{loc['route']}: {loc['path']}: only exact paths and a trailing * can be served by nginx")
+        key = (loc["match"], loc["path"])
+        if key in seen:
+            problems.append(f"{loc['route']}: {loc['path']} is also routed by {seen[key]}")
+        seen.setdefault(key, loc["route"])
+    for a in locations:
+        if a["match"] != "^~":
+            continue
+        for b in locations:
+            if b["service"] != a["service"] and b["path"].startswith(a["path"]):
+                problems.append(f"{a['route']}: {a['path']}* covers {b['route']}'s {b['path']}, nginx would not follow ALB priorities")
+    return problems
+
+
+class FilterModule:
+    def filters(self):
+        return {
+            "ooniapi_gateway_locations": ooniapi_gateway_locations,
+            "ooniapi_gateway_direct_hosts": ooniapi_gateway_direct_hosts,
+            "ooniapi_gateway_route_problems": ooniapi_gateway_route_problems,
+        }

ansible/roles/ooniapi_gateway/handlers/main.yml

@@ -0,0 +1,14 @@
+---
+- name: test ooniapi gateway nginx config
+  ansible.builtin.command: /usr/sbin/nginx -t -c /etc/nginx/nginx.conf
+  listen: reload ooniapi gateway nginx
+
+- name: reload ooniapi gateway nginx
+  ansible.builtin.service:
+    name: nginx
+    state: reloaded
+
+- name: restart docker
+  ansible.builtin.systemd_service:
+    name: docker
+    state: restarted