From 2f59fda0bc9f58179ccb1c623e2bf15384c1f78c Mon Sep 17 00:00:00 2001 From: AI Dev Date: Tue, 1 Sep 2026 18:03:11 +0000 Subject: [PATCH 1/4] Run Puma in cluster mode so production can use more than one core MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MRI executes Ruby on one core per process, so a single Puma process with sixteen threads still serves every request from one core while the rest of the machine idles. Two workers in production, overridable through WEB_CONCURRENCY; development stays in single mode. No preload_app!, because phased restarts (SIGUSR1) — the way to recycle workers without dropping requests — need workers that boot the app themselves. --- config/puma.rb | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/config/puma.rb b/config/puma.rb index 6946840..5932fd5 100644 --- a/config/puma.rb +++ b/config/puma.rb @@ -32,17 +32,17 @@ # Specifies the number of `workers` to boot in clustered mode. # Workers are forked web server processes. If using threads and workers together # the concurrency of the application would be max `threads` * `workers`. -# Workers do not work on JRuby or Windows (both of which do not support -# processes). # -# workers ENV.fetch("WEB_CONCURRENCY") { 2 } - -# Use the `preload_app!` method when specifying a `workers` number. -# This directive tells Puma to first boot the application and load code -# before forking the application. This takes advantage of Copy On Write -# process behavior so workers use less memory. -# -# preload_app! +# MRI executes Ruby on one core per process, so however many threads run above, +# a single process caps the whole app at one core. Two workers in production; +# development keeps single mode, where 0 means no cluster at all. +default_workers = ENV.fetch('RAILS_ENV', 'development') == 'production' ? 2 : 0 +workers ENV.fetch('WEB_CONCURRENCY') { default_workers }.to_i + +# No preload_app!: phased restarts (SIGUSR1) replace workers one at a time with +# the listener kept open — a restart without dropped requests — and they only +# work when each worker can boot the app itself rather than inherit it from a +# fork. # Allow puma to be restarted by `bin/rails restart` command. plugin :tmp_restart From ca2ea0bd4552a14861df916523697a6442d3e73c Mon Sep 17 00:00:00 2001 From: AI Dev Date: Tue, 1 Sep 2026 18:03:11 +0000 Subject: [PATCH 2/4] Give the nightly purge container a deadline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The rails runner that performs PurgeImagesJob can deadlock at process exit while draining the :async adapter's thread pool — ActiveStorage still enqueues PurgeJobs during destroy despite the inline purge. When that happens the container never exits, --rm never fires, one hung container accumulates per night holding a MySQL connection open, and set -e keeps every later step of the script (download retention, orphan reap, shard-directory cleanup) from running at all. The purge work itself finishes in seconds, before the hang, so bounding the container's lifetime at ten minutes loses nothing. --- deploy/purge-snapshots.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/deploy/purge-snapshots.sh b/deploy/purge-snapshots.sh index 5fb461d..fcccac3 100755 --- a/deploy/purge-snapshots.sh +++ b/deploy/purge-snapshots.sh @@ -36,13 +36,21 @@ log "purging snapshots past retention (image ${IMAGE_TAG:0:12})" # A one-off container rather than `docker exec` into web-prod: purging is IO # heavy and should not compete with request threads for the web process, and a # crash here must not take the site down. -docker run --rm \ +# --name + timeout + rm -f: the runner can deadlock at process exit draining +# the :async adapter's thread pool (ActiveStorage still enqueues PurgeJobs +# during destroy despite the inline purge). When that happens the container +# never exits, --rm never fires, one hung container accumulates per night +# holding a MySQL connection, and set -e stops every later step of this script +# from running. The purge work itself finishes in seconds, before the hang, so +# a bounded lifetime loses nothing; ten minutes is generous. +timeout 600 docker run --rm --name openipc-purge-snapshots \ --env-file /srv/www/.env.prod \ -v /run/mysqld:/run/mysqld \ -v "$BLOB_ROOT":/rails/storage \ "ghcr.io/openipc/website:${IMAGE_TAG}" \ bundle exec rails runner 'puts "purged #{PurgeImagesJob.new.perform} snapshots"' \ - || { log "FAILED: purge job errored"; exit 1; } + || log "WARNING: purge job errored or timed out, continuing" +docker rm -f openipc-purge-snapshots >/dev/null 2>&1 || true # Download rows, past their window. A row is about 60 bytes and the site sends # roughly eighty images a day, so two years of them is a few megabytes -- the From 1401ca7f1b9ed5a0d5fc885cf0ef2af824894cd3 Mon Sep 17 00:00:00 2001 From: AI Dev Date: Tue, 1 Sep 2026 18:09:11 +0000 Subject: [PATCH 3/4] Take a lock so purges cannot overlap The purge container carries a fixed name and the cleanup after it removes whatever currently holds that name, so two overlapping runs -- the cron one plus a manual invocation -- would have the newcomer kill the running purge's container and then skip its own purge while still executing the later cleanup steps. flock on a lock file makes the second invocation bow out instead. --- deploy/purge-snapshots.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/deploy/purge-snapshots.sh b/deploy/purge-snapshots.sh index fcccac3..2f68e96 100755 --- a/deploy/purge-snapshots.sh +++ b/deploy/purge-snapshots.sh @@ -14,6 +14,14 @@ set -euo pipefail +# One purge at a time. The container below carries a fixed name and the +# cleanup after it removes whatever holds that name, so an overlapping run -- +# cron plus a manual invocation, say -- would kill the other's container and +# then skip its own purge. The lock is released when the script exits and fd 9 +# closes. +exec 9>/var/lock/openipc-purge-snapshots.lock +flock -n 9 || { echo "another purge is already running; leaving it to finish"; exit 0; } + COMPOSE_DIR=/srv/www/deploy-src/deploy BLOB_ROOT=/srv/www/shared/storage IMAGE_TAG=$(sed -n 's/^PROD_TAG=//p' "${COMPOSE_DIR}/.env" | tail -1) From 74c0f766e9c7fcd89e67f36b3833fba8b56264fe Mon Sep 17 00:00:00 2001 From: AI Dev Date: Tue, 1 Sep 2026 18:19:15 +0000 Subject: [PATCH 4/4] Clear a stale purge container before running, not only after A container left behind by an interrupted run -- host reboot, script killed after the timeout fired but before its own cleanup -- would make the next night's docker run fail on the name collision, and that night's purge would be skipped with only a warning. Under the flock nothing legitimate can hold the name, so remove whatever does before starting. Raised by review on #133. --- deploy/purge-snapshots.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/deploy/purge-snapshots.sh b/deploy/purge-snapshots.sh index 2f68e96..9673e36 100755 --- a/deploy/purge-snapshots.sh +++ b/deploy/purge-snapshots.sh @@ -51,6 +51,11 @@ log "purging snapshots past retention (image ${IMAGE_TAG:0:12})" # holding a MySQL connection, and set -e stops every later step of this script # from running. The purge work itself finishes in seconds, before the hang, so # a bounded lifetime loses nothing; ten minutes is generous. +# A stale container from an interrupted run -- host reboot, script killed +# after the timeout fired but before its own cleanup -- would make this run +# fail on the name collision and cost a night's purge. Under the flock nothing +# legitimate holds this name, so clear it first. +docker rm -f openipc-purge-snapshots >/dev/null 2>&1 || true timeout 600 docker run --rm --name openipc-purge-snapshots \ --env-file /srv/www/.env.prod \ -v /run/mysqld:/run/mysqld \