From 41cb75f27947b609a51ad15900b675335613a66f Mon Sep 17 00:00:00 2001
From: Marco Acierno <marcoaciernoemail@gmail.com>
Date: Sun, 20 Sep 2026 20:11:34 +0200
Subject: [PATCH 1/4] Run Resonate and its worker on the ECS cluster

The durable workflows had nowhere to run outside a laptop. This adds both
halves: a Resonate server task on the cluster, next to redis and clamav,
and a `backend-resonate-worker` service running `manage.py
resonate_worker` off the backend image.

The server keeps its promises on the database instance the backend
already uses, in a database of its own. Terraform cannot create that
database (the instance is private), so a short container does it with
`createdb` before the server starts, on every deploy, doing nothing after
the first; Resonate creates its own schema from there.

Port 8001 is reachable only from the cluster's own security group, the
way redis and clamav are, and `RESONATE_URL` goes to every backend task:
the worker executes workflows, the admin starts them.
---
 infrastructure/applications/applications.tf   |   8 +
 .../applications/cluster/security.tf          |  19 +++
 .../pycon_backend/resonate_worker.tf          |  64 ++++++++
 .../applications/pycon_backend/worker.tf      |   5 +
 infrastructure/applications/resonate/task.tf  | 141 ++++++++++++++++++
 .../applications/resonate/variables.tf        |   3 +
 6 files changed, 240 insertions(+)
 create mode 100644 infrastructure/applications/pycon_backend/resonate_worker.tf
 create mode 100644 infrastructure/applications/resonate/task.tf
 create mode 100644 infrastructure/applications/resonate/variables.tf

diff --git a/infrastructure/applications/applications.tf b/infrastructure/applications/applications.tf
index a99c34c1f5..18f1a9eac7 100644
--- a/infrastructure/applications/applications.tf
+++ b/infrastructure/applications/applications.tf
@@ -39,6 +39,14 @@ module "pycon_frontend" {
   cf_hosted_zone_id = module.cluster.cf_hosted_zone_id
 }
 
+module "resonate" {
+  source          = "./resonate"
+  cluster_id      = module.cluster.cluster_id
+  logs_group_name = module.cluster.logs_group_name
+
+  database_settings = module.database.database_settings
+}
+
 module "clamav" {
   source       = "./clamav"
   cluster_id = module.cluster.cluster_id
diff --git a/infrastructure/applications/cluster/security.tf b/infrastructure/applications/cluster/security.tf
index 9c050a3231..f53dd6d37d 100644
--- a/infrastructure/applications/cluster/security.tf
+++ b/infrastructure/applications/cluster/security.tf
@@ -64,6 +64,25 @@ resource "aws_security_group_rule" "out_clamav" {
   security_group_id = aws_security_group.server.id
 }
 
+resource "aws_security_group_rule" "in_resonate" {
+  type              = "egress"
+  from_port         = 8001
+  to_port           = 8001
+  protocol          = "tcp"
+  source_security_group_id = aws_security_group.server.id
+  security_group_id = aws_security_group.server.id
+}
+
+resource "aws_security_group_rule" "out_resonate" {
+  # needed by fargate to connect to the server with resonate
+  type              = "ingress"
+  from_port         = 8001
+  to_port           = 8001
+  protocol          = "tcp"
+  source_security_group_id = aws_security_group.server.id
+  security_group_id = aws_security_group.server.id
+}
+
 resource "aws_security_group_rule" "web_http" {
   type              = "ingress"
   from_port         = 80
diff --git a/infrastructure/applications/pycon_backend/resonate_worker.tf b/infrastructure/applications/pycon_backend/resonate_worker.tf
new file mode 100644
index 0000000000..60d0a1045a
--- /dev/null
+++ b/infrastructure/applications/pycon_backend/resonate_worker.tf
@@ -0,0 +1,64 @@
+resource "aws_ecs_task_definition" "resonate_worker" {
+  family = "pythonit-${terraform.workspace}-resonate-worker"
+
+  container_definitions = jsonencode([
+    {
+      name              = "resonate-worker"
+      image             = "${data.aws_ecr_repository.be_repo.repository_url}@${data.aws_ecr_image.be_arm_image.image_digest}"
+      memoryReservation = local.is_prod ? 200 : 10
+      essential         = true
+
+      # Reloading is off outside DEBUG anyway; saying so keeps the deployed
+      # process a single one, with no reloader parent to sit between ECS and
+      # the worker's own signal handling.
+      command = [
+        "python", "manage.py", "resonate_worker", "--no-reload"
+      ]
+
+      environment = local.env_vars
+
+      mountPoints = []
+      systemControls = [
+        {
+          "namespace" : "net.core.somaxconn",
+          "value" : "4096"
+        }
+      ]
+
+      logConfiguration = {
+        logDriver = "awslogs"
+        options = {
+          "awslogs-group"         = var.logs_group_name
+          "awslogs-region"        = "eu-central-1"
+          "awslogs-stream-prefix" = "resonate-worker"
+        }
+      }
+
+      healthCheck = {
+        retries = 3
+        command = [
+          "CMD-SHELL",
+          "echo 1"
+        ]
+        timeout  = 3
+        interval = 10
+      }
+
+      # Long enough for the worker to hand its tasks back to the server
+      # instead of letting their leases lapse.
+      stopTimeout = 120
+    }
+  ])
+
+  requires_compatibilities = []
+  tags                     = {}
+}
+
+resource "aws_ecs_service" "resonate_worker" {
+  name                               = "backend-resonate-worker"
+  cluster                            = var.cluster_id
+  task_definition                    = aws_ecs_task_definition.resonate_worker.arn
+  desired_count                      = 1
+  deployment_minimum_healthy_percent = 0
+  deployment_maximum_percent         = 100
+}
diff --git a/infrastructure/applications/pycon_backend/worker.tf b/infrastructure/applications/pycon_backend/worker.tf
index 7af5ac89b0..e75aa8e391 100644
--- a/infrastructure/applications/pycon_backend/worker.tf
+++ b/infrastructure/applications/pycon_backend/worker.tf
@@ -188,6 +188,11 @@ locals {
     {
       name = "SNS_WEBHOOK_SECRET",
       value = module.common_secrets.value.sns_webhook_secret
+    },
+    {
+      # Both the workflows' worker and whoever starts one (the admin) need it.
+      name = "RESONATE_URL",
+      value = "http://${var.server_ip}:8001"
     }
   ]
 }
diff --git a/infrastructure/applications/resonate/task.tf b/infrastructure/applications/resonate/task.tf
new file mode 100644
index 0000000000..8f9f389ee8
--- /dev/null
+++ b/infrastructure/applications/resonate/task.tf
@@ -0,0 +1,141 @@
+locals {
+  is_prod = terraform.workspace == "production"
+
+  # Resonate keeps its promises on the database instance the backend already
+  # uses, in a database of its own.
+  database = "resonate"
+
+  database_url = "postgres://${var.database_settings.username}:${var.database_settings.password}@${var.database_settings.address}:${var.database_settings.port}/${local.database}?sslmode=require"
+}
+
+resource "aws_ecs_task_definition" "resonate" {
+  family = "pythonit-${terraform.workspace}-resonate"
+
+  container_definitions = jsonencode([
+    {
+      # The instance is private, so Terraform cannot create the database
+      # itself. This runs before the server on every deploy and does nothing
+      # after the first one.
+      name              = "create-database"
+      image             = "postgres:18-alpine"
+      memoryReservation = 10
+      essential         = false
+
+      command = [
+        "sh", "-c", "createdb ${local.database} || true"
+      ]
+
+      environment = [
+        {
+          name  = "PGHOST"
+          value = var.database_settings.address
+        },
+        {
+          name  = "PGPORT"
+          value = tostring(var.database_settings.port)
+        },
+        {
+          name  = "PGUSER"
+          value = var.database_settings.username
+        },
+        {
+          name  = "PGPASSWORD"
+          value = var.database_settings.password
+        },
+        {
+          # Where to connect to issue the CREATE DATABASE, not what to create.
+          name  = "PGDATABASE"
+          value = var.database_settings.db_name
+        },
+        {
+          name  = "PGSSLMODE"
+          value = "require"
+        }
+      ]
+
+      mountPoints    = []
+      systemControls = []
+
+      logConfiguration = {
+        logDriver = "awslogs"
+        options = {
+          "awslogs-group"         = var.logs_group_name
+          "awslogs-region"        = "eu-central-1"
+          "awslogs-stream-prefix" = "resonate-create-database"
+        }
+      }
+    },
+    {
+      name              = "resonate"
+      image             = "resonatehqio/resonate:v0.9.8"
+      memoryReservation = local.is_prod ? 200 : 10
+      essential         = true
+
+      dependsOn = [
+        {
+          containerName = "create-database"
+          condition     = "SUCCESS"
+        }
+      ]
+
+      environment = [
+        {
+          name  = "RESONATE_SERVER__BIND"
+          value = "0.0.0.0"
+        },
+        {
+          name  = "RESONATE_STORAGE__TYPE"
+          value = "postgres"
+        },
+        {
+          name  = "RESONATE_STORAGE__POSTGRES__URL"
+          value = local.database_url
+        }
+      ]
+
+      portMappings = [
+        {
+          containerPort = 8001
+          hostPort      = 8001
+          name          = "resonate"
+        }
+      ]
+
+      mountPoints    = []
+      systemControls = []
+
+      logConfiguration = {
+        logDriver = "awslogs"
+        options = {
+          "awslogs-group"         = var.logs_group_name
+          "awslogs-region"        = "eu-central-1"
+          "awslogs-stream-prefix" = "resonate"
+        }
+      }
+
+      healthCheck = {
+        retries = 3
+        command = [
+          "CMD-SHELL",
+          "wget -qO- http://127.0.0.1:8001/health || exit 1"
+        ]
+        timeout  = 3
+        interval = 10
+      }
+
+      stopTimeout = 300
+    }
+  ])
+
+  requires_compatibilities = []
+  tags                     = {}
+}
+
+resource "aws_ecs_service" "resonate" {
+  name                               = "resonate"
+  cluster                            = var.cluster_id
+  task_definition                    = aws_ecs_task_definition.resonate.arn
+  desired_count                      = 1
+  deployment_minimum_healthy_percent = 0
+  deployment_maximum_percent         = 100
+}
diff --git a/infrastructure/applications/resonate/variables.tf b/infrastructure/applications/resonate/variables.tf
new file mode 100644
index 0000000000..0d0ce653d4
--- /dev/null
+++ b/infrastructure/applications/resonate/variables.tf
@@ -0,0 +1,3 @@
+variable "cluster_id" {}
+variable "logs_group_name" {}
+variable "database_settings" {}

From f57f02e4cc877e612dd4c6b3abe957224551bab5 Mon Sep 17 00:00:00 2001
From: Marco Acierno <marcoaciernoemail@gmail.com>
Date: Sun, 20 Sep 2026 21:14:41 +0200
Subject: [PATCH 2/4] Match the module block style of its neighbours

The resonate module was the only one in applications.tf splitting its
inputs into two groups.
---
 infrastructure/applications/applications.tf | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/infrastructure/applications/applications.tf b/infrastructure/applications/applications.tf
index 18f1a9eac7..3edc952cdb 100644
--- a/infrastructure/applications/applications.tf
+++ b/infrastructure/applications/applications.tf
@@ -40,10 +40,9 @@ module "pycon_frontend" {
 }
 
 module "resonate" {
-  source          = "./resonate"
-  cluster_id      = module.cluster.cluster_id
+  source = "./resonate"
+  cluster_id = module.cluster.cluster_id
   logs_group_name = module.cluster.logs_group_name
-
   database_settings = module.database.database_settings
 }
 

From 693f5e4225245f4aa5b33e53f602c842626f6cf4 Mon Sep 17 00:00:00 2001
From: Marco Acierno <marcoaciernoemail@gmail.com>
Date: Sun, 20 Sep 2026 21:23:55 +0200
Subject: [PATCH 3/4] Make the Resonate database check mean something

Review follow-ups on the ECS setup.

`createdb ... || true` exited zero whatever happened, so the server's
SUCCESS dependency on it only promised that the container had run: a
wrong password or an unreachable instance was swallowed, the server
started anyway, and the only evidence sat in a log stream nobody would
think to open. Falling back to a `select 1` against the database instead
makes the container exit zero if and only if the database is there, and
stops it logging `already exists` on every task start.

The health check also had no start period and three retries, giving the
server ~30 seconds to come up -- the same 30 seconds it spends waiting on
the database pool before giving up, and before creating its schema on
first boot. It now gets the room docker-compose already gave it.
---
 infrastructure/applications/resonate/task.tf | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/infrastructure/applications/resonate/task.tf b/infrastructure/applications/resonate/task.tf
index 8f9f389ee8..75c82f1df8 100644
--- a/infrastructure/applications/resonate/task.tf
+++ b/infrastructure/applications/resonate/task.tf
@@ -15,14 +15,17 @@ resource "aws_ecs_task_definition" "resonate" {
     {
       # The instance is private, so Terraform cannot create the database
       # itself. This runs before the server on every deploy and does nothing
-      # after the first one.
+      # after the first one. It has to end up non-zero when the database is
+      # genuinely not there, or the server's SUCCESS dependency on it would
+      # mean nothing more than "the container ran".
       name              = "create-database"
       image             = "postgres:18-alpine"
       memoryReservation = 10
       essential         = false
 
       command = [
-        "sh", "-c", "createdb ${local.database} || true"
+        "sh", "-c",
+        "createdb ${local.database} 2>/dev/null || psql -d ${local.database} -c 'select 1' >/dev/null"
       ]
 
       environment = [
@@ -114,13 +117,17 @@ resource "aws_ecs_task_definition" "resonate" {
       }
 
       healthCheck = {
-        retries = 3
+        retries = 10
         command = [
           "CMD-SHELL",
           "wget -qO- http://127.0.0.1:8001/health || exit 1"
         ]
         timeout  = 3
         interval = 10
+        # The server spends up to 30 seconds waiting on the database pool
+        # before it gives up, and creates its schema on first boot, so it
+        # needs more room than the default none before checks start counting.
+        startPeriod = 60
       }
 
       stopTimeout = 300

From 514f7ffb8bbf7397c159fd6bb7051077c4c24fa6 Mon Sep 17 00:00:00 2001
From: Marco Acierno <marcoaciernoemail@gmail.com>
Date: Sun, 20 Sep 2026 21:36:16 +0200
Subject: [PATCH 4/4] Create the Resonate database outside Terraform

The bootstrap container is gone: the `resonate` database has to exist on
the instance before this deploys.
---
 infrastructure/applications/resonate/task.tf | 66 +-------------------
 1 file changed, 2 insertions(+), 64 deletions(-)

diff --git a/infrastructure/applications/resonate/task.tf b/infrastructure/applications/resonate/task.tf
index 75c82f1df8..f20fabae2e 100644
--- a/infrastructure/applications/resonate/task.tf
+++ b/infrastructure/applications/resonate/task.tf
@@ -2,7 +2,8 @@ locals {
   is_prod = terraform.workspace == "production"
 
   # Resonate keeps its promises on the database instance the backend already
-  # uses, in a database of its own.
+  # uses, in a database of its own. The instance is private, so that database
+  # is not created from here: it has to exist before this runs.
   database = "resonate"
 
   database_url = "postgres://${var.database_settings.username}:${var.database_settings.password}@${var.database_settings.address}:${var.database_settings.port}/${local.database}?sslmode=require"
@@ -12,75 +13,12 @@ resource "aws_ecs_task_definition" "resonate" {
   family = "pythonit-${terraform.workspace}-resonate"
 
   container_definitions = jsonencode([
-    {
-      # The instance is private, so Terraform cannot create the database
-      # itself. This runs before the server on every deploy and does nothing
-      # after the first one. It has to end up non-zero when the database is
-      # genuinely not there, or the server's SUCCESS dependency on it would
-      # mean nothing more than "the container ran".
-      name              = "create-database"
-      image             = "postgres:18-alpine"
-      memoryReservation = 10
-      essential         = false
-
-      command = [
-        "sh", "-c",
-        "createdb ${local.database} 2>/dev/null || psql -d ${local.database} -c 'select 1' >/dev/null"
-      ]
-
-      environment = [
-        {
-          name  = "PGHOST"
-          value = var.database_settings.address
-        },
-        {
-          name  = "PGPORT"
-          value = tostring(var.database_settings.port)
-        },
-        {
-          name  = "PGUSER"
-          value = var.database_settings.username
-        },
-        {
-          name  = "PGPASSWORD"
-          value = var.database_settings.password
-        },
-        {
-          # Where to connect to issue the CREATE DATABASE, not what to create.
-          name  = "PGDATABASE"
-          value = var.database_settings.db_name
-        },
-        {
-          name  = "PGSSLMODE"
-          value = "require"
-        }
-      ]
-
-      mountPoints    = []
-      systemControls = []
-
-      logConfiguration = {
-        logDriver = "awslogs"
-        options = {
-          "awslogs-group"         = var.logs_group_name
-          "awslogs-region"        = "eu-central-1"
-          "awslogs-stream-prefix" = "resonate-create-database"
-        }
-      }
-    },
     {
       name              = "resonate"
       image             = "resonatehqio/resonate:v0.9.8"
       memoryReservation = local.is_prod ? 200 : 10
       essential         = true
 
-      dependsOn = [
-        {
-          containerName = "create-database"
-          condition     = "SUCCESS"
-        }
-      ]
-
       environment = [
         {
           name  = "RESONATE_SERVER__BIND"