From b09d7ba0b0ef483903614a1f6a402ddac13834ca Mon Sep 17 00:00:00 2001
From: Aaron Gibson <aagbsn@extc.org>
Date: Tue, 6 Oct 2026 14:21:54 +0000
Subject: [PATCH] ec2: optional swapfile from cloud-init; 1 GiB on the
 clickhouse proxy

The clickhouse proxy is a t3a.nano with 512 MiB and no swap. On
2026-10-06 at 06:50 UTC unattended-upgrades (about 160 MiB resident)
filled memory; the kernel evicted nginx's pages and the proxy stalled
for 12 minutes until the OOM killer finished it off, taking fastpath and
the API's ClickHouse access down with it (OOPRIV-140).

swap_size_mb defaults to 0, which renders the cloud-config as the empty
string it was, so the other instances' launch templates do not change.
cloud-init only runs at first boot and user_data is in ignore_changes,
so the running proxy keeps the swapfile enabled on it by hand; a
rebuilt or replaced one gets it from here. swappiness 10 keeps nginx in
RAM unless memory runs short.
---
 tf/environments/prod/main.tf            |  2 ++
 tf/modules/ec2/main.tf                  |  2 +-
 tf/modules/ec2/templates/cloud-init.yml | 12 ++++++++++++
 tf/modules/ec2/variables.tf             |  8 +++++++-
 4 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf
index b8877903..3b6c7221 100644
--- a/tf/environments/prod/main.tf
+++ b/tf/environments/prod/main.tf
@@ -519,6 +519,8 @@ module "ooni_clickhouse_proxy" {
 
   key_name      = module.adm_iam_roles.oonidevops_key_name
   instance_type = "t3a.nano"
+  # 512 MiB without swap: unattended-upgrades OOMed it (OOPRIV-140)
+  swap_size_mb = 1024
 
   name = "oonickprx"
   ingress_rules = [{
diff --git a/tf/modules/ec2/main.tf b/tf/modules/ec2/main.tf
index bf1ba645..f5576edf 100644
--- a/tf/modules/ec2/main.tf
+++ b/tf/modules/ec2/main.tf
@@ -47,7 +47,7 @@ data "cloudinit_config" "ooni_ec2" {
   part {
     filename     = "init.cfg"
     content_type = "text/cloud-config"
-    content      = templatefile("${path.module}/templates/cloud-init.yml", {})
+    content      = templatefile("${path.module}/templates/cloud-init.yml", { swap_size_mb = var.swap_size_mb })
   }
 
 }
diff --git a/tf/modules/ec2/templates/cloud-init.yml b/tf/modules/ec2/templates/cloud-init.yml
index e69de29b..93d34229 100644
--- a/tf/modules/ec2/templates/cloud-init.yml
+++ b/tf/modules/ec2/templates/cloud-init.yml
@@ -0,0 +1,12 @@
+%{ if swap_size_mb > 0 ~}
+swap:
+  filename: /swapfile
+  size: ${swap_size_mb * 1048576}
+write_files:
+  - path: /etc/sysctl.d/99-swappiness.conf
+    content: |
+      vm.swappiness=10
+# systemd-sysctl ran before cloud-init wrote the file
+runcmd:
+  - [sysctl, -p, /etc/sysctl.d/99-swappiness.conf]
+%{ endif ~}
diff --git a/tf/modules/ec2/variables.tf b/tf/modules/ec2/variables.tf
index 9ac18e71..d84b14a7 100644
--- a/tf/modules/ec2/variables.tf
+++ b/tf/modules/ec2/variables.tf
@@ -80,4 +80,10 @@ variable "iam_instance_profile_name" {
   description = "Optional IAM instance profile name for the instance (e.g. YACE via IMDS)"
   default     = null
   type        = string
-}
\ No newline at end of file
+}
+
+variable "swap_size_mb" {
+  description = "Size of a swapfile created by cloud-init at first boot, in MiB. 0 means no swap. Only new instances get it: user_data changes are ignored on existing ones"
+  default     = 0
+  type        = number
+}