From 70a60ad983daca66e06c93b75ad8469424b6e3f5 Mon Sep 17 00:00:00 2001 From: rfay Date: Thu, 6 Aug 2026 21:18:39 +0000 Subject: [PATCH] fix(startup): self-heal nested Docker daemon after in-place container restart When the host's docker-ce package is upgraded (or the host reboots), Docker restarts each workspace container in place via `restart = unless-stopped` rather than recreating it. The nested `dockerd` inside runs as a bare background process with nothing managing its lifecycle, so it gets killed uncleanly, leaving a stale /var/run/docker.pid that makes the next `dockerd` refuse to start ("process with PID ... is still running") even though nothing is running. Startup only checked for the socket *file*, not daemon health, so the workspace reported healthy while Docker (and therefore DDEV) stayed broken until someone manually restarted the workspace. Verified end-to-end against staging-coder.ddev.com (d12-1): both a host-level `docker-ce` upgrade and a full host reboot now recover on their own. Fix has two halves, extracted into a shared modules/docker-daemon module (vendored into each template dir like claude-remote-control) since the logic was identical across all three templates: - Shutdown: gracefully SIGTERM (then SIGKILL after a timeout) the nested dockerd before the container is torn down, so it doesn't leave corrupted/stale state behind in the first place. - Startup: verify actual daemon health via `docker info`, not just socket presence; clear stale pidfile/socket and retry once before giving up, so a container that does get killed uncleanly (e.g. by a hard host power-cut) still recovers without manual intervention. Co-Authored-By: Claude Sonnet 5 --- Makefile | 11 +++- drupal-contrib/.terraform.lock.hcl | 3 + drupal-contrib/modules/docker-daemon/main.tf | 49 +++++++++++++++++ .../modules/docker-daemon/outputs.tf | 55 +++++++++++++++++++ .../modules/docker-daemon/variables.tf | 4 ++ .../modules/docker-daemon/versions.tf | 7 +++ drupal-contrib/template.tf | 45 ++------------- drupal-core/.terraform.lock.hcl | 1 + drupal-core/modules/docker-daemon/main.tf | 49 +++++++++++++++++ drupal-core/modules/docker-daemon/outputs.tf | 55 +++++++++++++++++++ .../modules/docker-daemon/variables.tf | 4 ++ drupal-core/modules/docker-daemon/versions.tf | 7 +++ drupal-core/template.tf | 52 ++---------------- freeform/.terraform.lock.hcl | 1 + freeform/modules/docker-daemon/main.tf | 49 +++++++++++++++++ freeform/modules/docker-daemon/outputs.tf | 55 +++++++++++++++++++ freeform/modules/docker-daemon/variables.tf | 4 ++ freeform/modules/docker-daemon/versions.tf | 7 +++ freeform/template.tf | 46 ++-------------- modules/docker-daemon/main.tf | 49 +++++++++++++++++ modules/docker-daemon/outputs.tf | 55 +++++++++++++++++++ modules/docker-daemon/variables.tf | 4 ++ modules/docker-daemon/versions.tf | 7 +++ 23 files changed, 487 insertions(+), 132 deletions(-) create mode 100644 drupal-contrib/modules/docker-daemon/main.tf create mode 100644 drupal-contrib/modules/docker-daemon/outputs.tf create mode 100644 drupal-contrib/modules/docker-daemon/variables.tf create mode 100644 drupal-contrib/modules/docker-daemon/versions.tf create mode 100644 drupal-core/modules/docker-daemon/main.tf create mode 100644 drupal-core/modules/docker-daemon/outputs.tf create mode 100644 drupal-core/modules/docker-daemon/variables.tf create mode 100644 drupal-core/modules/docker-daemon/versions.tf create mode 100644 freeform/modules/docker-daemon/main.tf create mode 100644 freeform/modules/docker-daemon/outputs.tf create mode 100644 freeform/modules/docker-daemon/variables.tf create mode 100644 freeform/modules/docker-daemon/versions.tf create mode 100644 modules/docker-daemon/main.tf create mode 100644 modules/docker-daemon/outputs.tf create mode 100644 modules/docker-daemon/variables.tf create mode 100644 modules/docker-daemon/versions.tf diff --git a/Makefile b/Makefile index 4b1e2b1..fa5e07c 100644 --- a/Makefile +++ b/Makefile @@ -119,8 +119,17 @@ sync-vscode-extensions: ## Vendor shared/vscode-extensions.tf into each template done @echo "Synced shared/vscode-extensions.tf into: $(TEMPLATES)" +.PHONY: sync-docker-daemon-module +sync-docker-daemon-module: ## Vendor modules/docker-daemon into each template dir (same reason as sync-claude-module: push only bundles the given --directory) + @for t in $(TEMPLATES); do \ + rm -rf $$t/modules/docker-daemon; \ + mkdir -p $$t/modules; \ + cp -r modules/docker-daemon $$t/modules/docker-daemon; \ + done + @echo "Synced modules/docker-daemon into: $(TEMPLATES)" + .PHONY: sync-shared -sync-shared: sync-claude-module sync-vscode-extensions ## Vendor all shared Terraform assets (modules + vscode-extensions.tf) into each template dir +sync-shared: sync-claude-module sync-vscode-extensions sync-docker-daemon-module ## Vendor all shared Terraform assets (modules + vscode-extensions.tf) into each template dir .PHONY: validate validate: sync-shared ## Validate all Terraform templates (requires terraform in PATH) diff --git a/drupal-contrib/.terraform.lock.hcl b/drupal-contrib/.terraform.lock.hcl index 483d65d..2d9e3dd 100644 --- a/drupal-contrib/.terraform.lock.hcl +++ b/drupal-contrib/.terraform.lock.hcl @@ -6,6 +6,7 @@ provider "registry.terraform.io/coder/coder" { constraints = ">= 2.5.0, >= 2.13.0" hashes = [ "h1:GZ71aeLqkBjTfALHYIf9dWF4WDnRALPDMU5++8GY0Y8=", + "h1:bzQUVnn1l58ZSqgqQFEtPwzvtvn4+ExsX+8fkvqXlOo=", "zh:04c070bc17816ff4fb785a57c5d217c4c81f0c564cc6634a0c635e079fb68393", "zh:15b70d49e8a1fcab72ec9497ab7af90094a476bee8039a10aecdae2b05e644c1", "zh:24a731d6f94e3711a6f8f88995a0389e4efc96f926fcd97b5eed2c21e0481302", @@ -27,6 +28,7 @@ provider "registry.terraform.io/hashicorp/null" { version = "3.2.4" hashes = [ "h1:L5V05xwp/Gto1leRryuesxjMfgZwjb7oool4WS1UEFQ=", + "h1:hkf5w5B6q8e2A42ND2CjAvgvSN3puAosDmOJb3zCVQM=", "zh:59f6b52ab4ff35739647f9509ee6d93d7c032985d9f8c6237d1f8a59471bbbe2", "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", "zh:795c897119ff082133150121d39ff26cb5f89a730a2c8c26f3a9c1abf81a9c43", @@ -46,6 +48,7 @@ provider "registry.terraform.io/kreuzwerker/docker" { version = "3.9.0" constraints = "~> 3.0" hashes = [ + "h1:EAdNh5KgGPJT5jm848MRIfNfHUVJeTBdKKcFLax5g38=", "h1:p65AjYSOmmHPjKkIYlEnxSMyrprTHKf1qBzKhCnCtG8=", "zh:0ead8281830e9b9496651282235d9a139ba1b1b6ff79e395eb8c78658dc446b9", "zh:0f17d37d8d3872df3fb75c68b5272e0c981343f53b506a9675b4405191edd3ef", diff --git a/drupal-contrib/modules/docker-daemon/main.tf b/drupal-contrib/modules/docker-daemon/main.tf new file mode 100644 index 0000000..a06e49b --- /dev/null +++ b/drupal-contrib/modules/docker-daemon/main.tf @@ -0,0 +1,49 @@ +# Stops the nested Docker daemon (Sysbox) cleanly on workspace stop. It runs +# as a bare background process in the caller's startup script (see +# outputs.tf) -- no init system inside the container manages it -- so +# nothing else sends it a graceful shutdown before the container itself is +# torn down. That matters beyond a normal workspace stop: when something +# outside Coder restarts the container in place (e.g. the host's docker-ce +# package being upgraded, which restarts docker.service and, via `restart = +# unless-stopped`, restarts every workspace container), an unclean kill here +# leaves a stale /var/run/docker.pid behind that blocks the next dockerd +# from starting until the workspace is rebuilt. +resource "coder_script" "ddev_shutdown" { + agent_id = var.agent_id + display_name = "Stop DDEV Projects" + icon = "/icon/docker.svg" + run_on_stop = true + script = <<-EOT + #!/bin/bash + export PATH="$PATH:/home/linuxbrew/.linuxbrew/bin:/usr/local/bin" + # Wait for Docker socket — it should already be up, but guard against + # race conditions during workspace stop/update. + for i in $(seq 1 10); do + [ -S /var/run/docker.sock ] && break + sleep 1 + done + if [ -S /var/run/docker.sock ]; then + echo "Running ddev poweroff..." + ddev poweroff || true + echo "ddev poweroff complete" + else + echo "Docker socket not available; skipping ddev poweroff" + fi + + DOCKERD_PID=$(pgrep -x dockerd || true) + if [ -n "$DOCKERD_PID" ]; then + echo "Stopping Docker daemon (pid $DOCKERD_PID)..." + sudo kill -TERM "$DOCKERD_PID" 2>/dev/null || true + for i in $(seq 1 30); do + kill -0 "$DOCKERD_PID" 2>/dev/null || break + sleep 1 + done + if kill -0 "$DOCKERD_PID" 2>/dev/null; then + echo "Docker daemon did not stop in time; forcing" + sudo kill -KILL "$DOCKERD_PID" 2>/dev/null || true + else + echo "Docker daemon stopped cleanly" + fi + fi + EOT +} diff --git a/drupal-contrib/modules/docker-daemon/outputs.tf b/drupal-contrib/modules/docker-daemon/outputs.tf new file mode 100644 index 0000000..c7d1f96 --- /dev/null +++ b/drupal-contrib/modules/docker-daemon/outputs.tf @@ -0,0 +1,55 @@ +output "startup_script" { + description = "Bash snippet for the caller to append to its coder_agent startup_script, after registry-mirror configuration and before any command that needs Docker (ddev config global, image pre-pull, etc)." + value = <<-EOT + + # Start Docker Daemon (Sysbox). dockerd runs as a bare background + # process here (no init system inside the container manages it), so + # when something outside Coder restarts this same container in place + # -- e.g. the host's docker-ce package being upgraded, which restarts + # docker.service and, via `restart = unless-stopped`, restarts every + # workspace container -- dockerd is killed without a chance to clean + # up (see modules/docker-daemon for the matching graceful-shutdown + # half of this). That leaves a stale /var/run/docker.pid (and socket + # file) in the container's writable layer, which makes the next + # dockerd refuse to start ("process with PID ... is still running") + # even though nothing is actually running. Clear that leftover state + # before starting, and confirm the daemon actually answers rather than + # trusting the socket file alone. + wait_for_dockerd() { + for i in $(seq 1 30); do + sudo docker info > /dev/null 2>&1 && return 0 + sleep 1 + done + return 1 + } + + if sudo docker info > /dev/null 2>&1; then + echo "Docker Daemon already running." + else + sudo pkill -x dockerd 2>/dev/null || true + sudo rm -f /var/run/docker.pid /var/run/docker.sock + + echo "Starting Docker Daemon..." + sudo dockerd > /tmp/dockerd.log 2>&1 & + if wait_for_dockerd; then + echo "Docker Daemon ready" + else + echo "Docker Daemon not responding after 30s; retrying after cleanup" + sudo pkill -x dockerd 2>/dev/null || true + sleep 2 + sudo rm -f /var/run/docker.pid /var/run/docker.sock + sudo dockerd > /tmp/dockerd.log 2>&1 & + if wait_for_dockerd; then + echo "Docker Daemon ready after retry" + else + echo "Error: Docker Daemon failed to start after cleanup + retry; see /tmp/dockerd.log" + tail -n 40 /tmp/dockerd.log || true + fi + fi + fi + + if [ -S /var/run/docker.sock ]; then + sudo chmod 666 /var/run/docker.sock + fi + EOT +} diff --git a/drupal-contrib/modules/docker-daemon/variables.tf b/drupal-contrib/modules/docker-daemon/variables.tf new file mode 100644 index 0000000..f9fb4c9 --- /dev/null +++ b/drupal-contrib/modules/docker-daemon/variables.tf @@ -0,0 +1,4 @@ +variable "agent_id" { + description = "ID of the coder_agent to attach the Docker daemon shutdown script to" + type = string +} diff --git a/drupal-contrib/modules/docker-daemon/versions.tf b/drupal-contrib/modules/docker-daemon/versions.tf new file mode 100644 index 0000000..394c7cb --- /dev/null +++ b/drupal-contrib/modules/docker-daemon/versions.tf @@ -0,0 +1,7 @@ +terraform { + required_providers { + coder = { + source = "coder/coder" + } + } +} diff --git a/drupal-contrib/template.tf b/drupal-contrib/template.tf index c5cdccd..86c8547 100644 --- a/drupal-contrib/template.tf +++ b/drupal-contrib/template.tf @@ -407,27 +407,7 @@ resource "coder_agent" "main" { EOF fi - if ! pgrep -x "dockerd" > /dev/null; then - echo "Starting Docker Daemon..." - sudo dockerd > /tmp/dockerd.log 2>&1 & - - echo "Waiting for Docker Socket..." - for i in $(seq 1 30); do - if [ -S /var/run/docker.sock ]; then - echo "Docker Socket found!" - break - fi - sleep 1 - done - - if [ -S /var/run/docker.sock ]; then - sudo chmod 666 /var/run/docker.sock - else - echo "Error: Docker Socket not found after 30s!" - fi - else - echo "Docker Daemon already running." - fi + ${module.docker_daemon.startup_script} mkdir -p ~/.ddev echo "Configuring DDEV to omit ddev-router..." @@ -1160,26 +1140,9 @@ resource "coder_app" "mailpit" { } } -resource "coder_script" "ddev_shutdown" { - agent_id = coder_agent.main.id - display_name = "Stop DDEV Projects" - icon = "/icon/docker.svg" - run_on_stop = true - script = <<-EOT - #!/bin/bash - export PATH="$PATH:/home/linuxbrew/.linuxbrew/bin:/usr/local/bin" - for i in $(seq 1 10); do - [ -S /var/run/docker.sock ] && break - sleep 1 - done - if [ ! -S /var/run/docker.sock ]; then - echo "Docker socket not available; skipping ddev poweroff" - exit 0 - fi - echo "Running ddev poweroff..." - ddev poweroff || true - echo "ddev poweroff complete" - EOT +module "docker_daemon" { + source = "./modules/docker-daemon" + agent_id = coder_agent.main.id } resource "docker_container" "workspace" { diff --git a/drupal-core/.terraform.lock.hcl b/drupal-core/.terraform.lock.hcl index 684d6df..cd1225a 100644 --- a/drupal-core/.terraform.lock.hcl +++ b/drupal-core/.terraform.lock.hcl @@ -29,6 +29,7 @@ provider "registry.terraform.io/hashicorp/null" { version = "3.2.4" hashes = [ "h1:L5V05xwp/Gto1leRryuesxjMfgZwjb7oool4WS1UEFQ=", + "h1:hkf5w5B6q8e2A42ND2CjAvgvSN3puAosDmOJb3zCVQM=", "zh:59f6b52ab4ff35739647f9509ee6d93d7c032985d9f8c6237d1f8a59471bbbe2", "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", "zh:795c897119ff082133150121d39ff26cb5f89a730a2c8c26f3a9c1abf81a9c43", diff --git a/drupal-core/modules/docker-daemon/main.tf b/drupal-core/modules/docker-daemon/main.tf new file mode 100644 index 0000000..a06e49b --- /dev/null +++ b/drupal-core/modules/docker-daemon/main.tf @@ -0,0 +1,49 @@ +# Stops the nested Docker daemon (Sysbox) cleanly on workspace stop. It runs +# as a bare background process in the caller's startup script (see +# outputs.tf) -- no init system inside the container manages it -- so +# nothing else sends it a graceful shutdown before the container itself is +# torn down. That matters beyond a normal workspace stop: when something +# outside Coder restarts the container in place (e.g. the host's docker-ce +# package being upgraded, which restarts docker.service and, via `restart = +# unless-stopped`, restarts every workspace container), an unclean kill here +# leaves a stale /var/run/docker.pid behind that blocks the next dockerd +# from starting until the workspace is rebuilt. +resource "coder_script" "ddev_shutdown" { + agent_id = var.agent_id + display_name = "Stop DDEV Projects" + icon = "/icon/docker.svg" + run_on_stop = true + script = <<-EOT + #!/bin/bash + export PATH="$PATH:/home/linuxbrew/.linuxbrew/bin:/usr/local/bin" + # Wait for Docker socket — it should already be up, but guard against + # race conditions during workspace stop/update. + for i in $(seq 1 10); do + [ -S /var/run/docker.sock ] && break + sleep 1 + done + if [ -S /var/run/docker.sock ]; then + echo "Running ddev poweroff..." + ddev poweroff || true + echo "ddev poweroff complete" + else + echo "Docker socket not available; skipping ddev poweroff" + fi + + DOCKERD_PID=$(pgrep -x dockerd || true) + if [ -n "$DOCKERD_PID" ]; then + echo "Stopping Docker daemon (pid $DOCKERD_PID)..." + sudo kill -TERM "$DOCKERD_PID" 2>/dev/null || true + for i in $(seq 1 30); do + kill -0 "$DOCKERD_PID" 2>/dev/null || break + sleep 1 + done + if kill -0 "$DOCKERD_PID" 2>/dev/null; then + echo "Docker daemon did not stop in time; forcing" + sudo kill -KILL "$DOCKERD_PID" 2>/dev/null || true + else + echo "Docker daemon stopped cleanly" + fi + fi + EOT +} diff --git a/drupal-core/modules/docker-daemon/outputs.tf b/drupal-core/modules/docker-daemon/outputs.tf new file mode 100644 index 0000000..c7d1f96 --- /dev/null +++ b/drupal-core/modules/docker-daemon/outputs.tf @@ -0,0 +1,55 @@ +output "startup_script" { + description = "Bash snippet for the caller to append to its coder_agent startup_script, after registry-mirror configuration and before any command that needs Docker (ddev config global, image pre-pull, etc)." + value = <<-EOT + + # Start Docker Daemon (Sysbox). dockerd runs as a bare background + # process here (no init system inside the container manages it), so + # when something outside Coder restarts this same container in place + # -- e.g. the host's docker-ce package being upgraded, which restarts + # docker.service and, via `restart = unless-stopped`, restarts every + # workspace container -- dockerd is killed without a chance to clean + # up (see modules/docker-daemon for the matching graceful-shutdown + # half of this). That leaves a stale /var/run/docker.pid (and socket + # file) in the container's writable layer, which makes the next + # dockerd refuse to start ("process with PID ... is still running") + # even though nothing is actually running. Clear that leftover state + # before starting, and confirm the daemon actually answers rather than + # trusting the socket file alone. + wait_for_dockerd() { + for i in $(seq 1 30); do + sudo docker info > /dev/null 2>&1 && return 0 + sleep 1 + done + return 1 + } + + if sudo docker info > /dev/null 2>&1; then + echo "Docker Daemon already running." + else + sudo pkill -x dockerd 2>/dev/null || true + sudo rm -f /var/run/docker.pid /var/run/docker.sock + + echo "Starting Docker Daemon..." + sudo dockerd > /tmp/dockerd.log 2>&1 & + if wait_for_dockerd; then + echo "Docker Daemon ready" + else + echo "Docker Daemon not responding after 30s; retrying after cleanup" + sudo pkill -x dockerd 2>/dev/null || true + sleep 2 + sudo rm -f /var/run/docker.pid /var/run/docker.sock + sudo dockerd > /tmp/dockerd.log 2>&1 & + if wait_for_dockerd; then + echo "Docker Daemon ready after retry" + else + echo "Error: Docker Daemon failed to start after cleanup + retry; see /tmp/dockerd.log" + tail -n 40 /tmp/dockerd.log || true + fi + fi + fi + + if [ -S /var/run/docker.sock ]; then + sudo chmod 666 /var/run/docker.sock + fi + EOT +} diff --git a/drupal-core/modules/docker-daemon/variables.tf b/drupal-core/modules/docker-daemon/variables.tf new file mode 100644 index 0000000..f9fb4c9 --- /dev/null +++ b/drupal-core/modules/docker-daemon/variables.tf @@ -0,0 +1,4 @@ +variable "agent_id" { + description = "ID of the coder_agent to attach the Docker daemon shutdown script to" + type = string +} diff --git a/drupal-core/modules/docker-daemon/versions.tf b/drupal-core/modules/docker-daemon/versions.tf new file mode 100644 index 0000000..394c7cb --- /dev/null +++ b/drupal-core/modules/docker-daemon/versions.tf @@ -0,0 +1,7 @@ +terraform { + required_providers { + coder = { + source = "coder/coder" + } + } +} diff --git a/drupal-core/template.tf b/drupal-core/template.tf index 9473e9f..53f97b0 100644 --- a/drupal-core/template.tf +++ b/drupal-core/template.tf @@ -484,32 +484,7 @@ resource "coder_agent" "main" { EOF fi - # Start Docker Daemon (Sysbox) - # Since we are not booting with systemd as PID 1, we must start dockerd manually. - if ! pgrep -x "dockerd" > /dev/null; then - echo "Starting Docker Daemon..." - # Use sudo because we are running as coder user - sudo dockerd > /tmp/dockerd.log 2>&1 & - - # Wait for Docker Socket - echo "Waiting for Docker Socket..." - for i in $(seq 1 30); do - if [ -S /var/run/docker.sock ]; then - echo "Docker Socket found!" - break - fi - sleep 1 - done - - # Fix permissions so 'coder' user can access it - if [ -S /var/run/docker.sock ]; then - sudo chmod 666 /var/run/docker.sock - else - echo "Error: Docker Socket not found after 30s!" - fi - else - echo "Docker Daemon already running." - fi + ${module.docker_daemon.startup_script} # Create .ddev directory for ddev config (DDEV creates global_config.yaml on first use) mkdir -p ~/.ddev @@ -1461,28 +1436,9 @@ resource "coder_app" "mailpit" { # No explicit app definitions needed - coder-login module enables Gateway support # Graceful DDEV shutdown when workspace stops -resource "coder_script" "ddev_shutdown" { - agent_id = coder_agent.main.id - display_name = "Stop DDEV Projects" - icon = "/icon/docker.svg" - run_on_stop = true - script = <<-EOT - #!/bin/bash - export PATH="$PATH:/home/linuxbrew/.linuxbrew/bin:/usr/local/bin" - # Wait for Docker socket — it should already be up, but guard against - # race conditions during workspace stop/update. - for i in $(seq 1 10); do - [ -S /var/run/docker.sock ] && break - sleep 1 - done - if [ ! -S /var/run/docker.sock ]; then - echo "Docker socket not available; skipping ddev poweroff" - exit 0 - fi - echo "Running ddev poweroff..." - ddev poweroff || true - echo "ddev poweroff complete" - EOT +module "docker_daemon" { + source = "./modules/docker-daemon" + agent_id = coder_agent.main.id } diff --git a/freeform/.terraform.lock.hcl b/freeform/.terraform.lock.hcl index 684d6df..cd1225a 100644 --- a/freeform/.terraform.lock.hcl +++ b/freeform/.terraform.lock.hcl @@ -29,6 +29,7 @@ provider "registry.terraform.io/hashicorp/null" { version = "3.2.4" hashes = [ "h1:L5V05xwp/Gto1leRryuesxjMfgZwjb7oool4WS1UEFQ=", + "h1:hkf5w5B6q8e2A42ND2CjAvgvSN3puAosDmOJb3zCVQM=", "zh:59f6b52ab4ff35739647f9509ee6d93d7c032985d9f8c6237d1f8a59471bbbe2", "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", "zh:795c897119ff082133150121d39ff26cb5f89a730a2c8c26f3a9c1abf81a9c43", diff --git a/freeform/modules/docker-daemon/main.tf b/freeform/modules/docker-daemon/main.tf new file mode 100644 index 0000000..a06e49b --- /dev/null +++ b/freeform/modules/docker-daemon/main.tf @@ -0,0 +1,49 @@ +# Stops the nested Docker daemon (Sysbox) cleanly on workspace stop. It runs +# as a bare background process in the caller's startup script (see +# outputs.tf) -- no init system inside the container manages it -- so +# nothing else sends it a graceful shutdown before the container itself is +# torn down. That matters beyond a normal workspace stop: when something +# outside Coder restarts the container in place (e.g. the host's docker-ce +# package being upgraded, which restarts docker.service and, via `restart = +# unless-stopped`, restarts every workspace container), an unclean kill here +# leaves a stale /var/run/docker.pid behind that blocks the next dockerd +# from starting until the workspace is rebuilt. +resource "coder_script" "ddev_shutdown" { + agent_id = var.agent_id + display_name = "Stop DDEV Projects" + icon = "/icon/docker.svg" + run_on_stop = true + script = <<-EOT + #!/bin/bash + export PATH="$PATH:/home/linuxbrew/.linuxbrew/bin:/usr/local/bin" + # Wait for Docker socket — it should already be up, but guard against + # race conditions during workspace stop/update. + for i in $(seq 1 10); do + [ -S /var/run/docker.sock ] && break + sleep 1 + done + if [ -S /var/run/docker.sock ]; then + echo "Running ddev poweroff..." + ddev poweroff || true + echo "ddev poweroff complete" + else + echo "Docker socket not available; skipping ddev poweroff" + fi + + DOCKERD_PID=$(pgrep -x dockerd || true) + if [ -n "$DOCKERD_PID" ]; then + echo "Stopping Docker daemon (pid $DOCKERD_PID)..." + sudo kill -TERM "$DOCKERD_PID" 2>/dev/null || true + for i in $(seq 1 30); do + kill -0 "$DOCKERD_PID" 2>/dev/null || break + sleep 1 + done + if kill -0 "$DOCKERD_PID" 2>/dev/null; then + echo "Docker daemon did not stop in time; forcing" + sudo kill -KILL "$DOCKERD_PID" 2>/dev/null || true + else + echo "Docker daemon stopped cleanly" + fi + fi + EOT +} diff --git a/freeform/modules/docker-daemon/outputs.tf b/freeform/modules/docker-daemon/outputs.tf new file mode 100644 index 0000000..c7d1f96 --- /dev/null +++ b/freeform/modules/docker-daemon/outputs.tf @@ -0,0 +1,55 @@ +output "startup_script" { + description = "Bash snippet for the caller to append to its coder_agent startup_script, after registry-mirror configuration and before any command that needs Docker (ddev config global, image pre-pull, etc)." + value = <<-EOT + + # Start Docker Daemon (Sysbox). dockerd runs as a bare background + # process here (no init system inside the container manages it), so + # when something outside Coder restarts this same container in place + # -- e.g. the host's docker-ce package being upgraded, which restarts + # docker.service and, via `restart = unless-stopped`, restarts every + # workspace container -- dockerd is killed without a chance to clean + # up (see modules/docker-daemon for the matching graceful-shutdown + # half of this). That leaves a stale /var/run/docker.pid (and socket + # file) in the container's writable layer, which makes the next + # dockerd refuse to start ("process with PID ... is still running") + # even though nothing is actually running. Clear that leftover state + # before starting, and confirm the daemon actually answers rather than + # trusting the socket file alone. + wait_for_dockerd() { + for i in $(seq 1 30); do + sudo docker info > /dev/null 2>&1 && return 0 + sleep 1 + done + return 1 + } + + if sudo docker info > /dev/null 2>&1; then + echo "Docker Daemon already running." + else + sudo pkill -x dockerd 2>/dev/null || true + sudo rm -f /var/run/docker.pid /var/run/docker.sock + + echo "Starting Docker Daemon..." + sudo dockerd > /tmp/dockerd.log 2>&1 & + if wait_for_dockerd; then + echo "Docker Daemon ready" + else + echo "Docker Daemon not responding after 30s; retrying after cleanup" + sudo pkill -x dockerd 2>/dev/null || true + sleep 2 + sudo rm -f /var/run/docker.pid /var/run/docker.sock + sudo dockerd > /tmp/dockerd.log 2>&1 & + if wait_for_dockerd; then + echo "Docker Daemon ready after retry" + else + echo "Error: Docker Daemon failed to start after cleanup + retry; see /tmp/dockerd.log" + tail -n 40 /tmp/dockerd.log || true + fi + fi + fi + + if [ -S /var/run/docker.sock ]; then + sudo chmod 666 /var/run/docker.sock + fi + EOT +} diff --git a/freeform/modules/docker-daemon/variables.tf b/freeform/modules/docker-daemon/variables.tf new file mode 100644 index 0000000..f9fb4c9 --- /dev/null +++ b/freeform/modules/docker-daemon/variables.tf @@ -0,0 +1,4 @@ +variable "agent_id" { + description = "ID of the coder_agent to attach the Docker daemon shutdown script to" + type = string +} diff --git a/freeform/modules/docker-daemon/versions.tf b/freeform/modules/docker-daemon/versions.tf new file mode 100644 index 0000000..394c7cb --- /dev/null +++ b/freeform/modules/docker-daemon/versions.tf @@ -0,0 +1,7 @@ +terraform { + required_providers { + coder = { + source = "coder/coder" + } + } +} diff --git a/freeform/template.tf b/freeform/template.tf index 04c40e3..76fd352 100644 --- a/freeform/template.tf +++ b/freeform/template.tf @@ -304,26 +304,7 @@ resource "coder_agent" "main" { EOF fi - # Start Docker Daemon (Sysbox) - if ! pgrep -x "dockerd" > /dev/null; then - echo "Starting Docker Daemon..." - sudo dockerd > /tmp/dockerd.log 2>&1 & - echo "Waiting for Docker socket..." - for i in $(seq 1 30); do - if [ -S /var/run/docker.sock ]; then - echo "Docker socket ready" - break - fi - sleep 1 - done - if [ -S /var/run/docker.sock ]; then - sudo chmod 666 /var/run/docker.sock - else - echo "Error: Docker socket not found after 30s" - fi - else - echo "Docker Daemon already running." - fi + ${module.docker_daemon.startup_script} # Configure DDEV global settings now that Docker is up (ddev config global needs Docker) ddev config global --instrumentation-opt-in=true > /dev/null 2>&1 || true @@ -552,28 +533,9 @@ resource "coder_app" "adminer" { } } -resource "coder_script" "ddev_shutdown" { - agent_id = coder_agent.main.id - display_name = "Stop DDEV Projects" - icon = "/icon/docker.svg" - run_on_stop = true - script = <<-EOT - #!/bin/bash - export PATH="$PATH:/home/linuxbrew/.linuxbrew/bin:/usr/local/bin" - # Wait for Docker socket — it should already be up, but guard against - # race conditions during workspace stop/update. - for i in $(seq 1 10); do - [ -S /var/run/docker.sock ] && break - sleep 1 - done - if [ ! -S /var/run/docker.sock ]; then - echo "Docker socket not available; skipping ddev poweroff" - exit 0 - fi - echo "Running ddev poweroff..." - ddev poweroff || true - echo "ddev poweroff complete" - EOT +module "docker_daemon" { + source = "./modules/docker-daemon" + agent_id = coder_agent.main.id } resource "docker_container" "workspace" { diff --git a/modules/docker-daemon/main.tf b/modules/docker-daemon/main.tf new file mode 100644 index 0000000..a06e49b --- /dev/null +++ b/modules/docker-daemon/main.tf @@ -0,0 +1,49 @@ +# Stops the nested Docker daemon (Sysbox) cleanly on workspace stop. It runs +# as a bare background process in the caller's startup script (see +# outputs.tf) -- no init system inside the container manages it -- so +# nothing else sends it a graceful shutdown before the container itself is +# torn down. That matters beyond a normal workspace stop: when something +# outside Coder restarts the container in place (e.g. the host's docker-ce +# package being upgraded, which restarts docker.service and, via `restart = +# unless-stopped`, restarts every workspace container), an unclean kill here +# leaves a stale /var/run/docker.pid behind that blocks the next dockerd +# from starting until the workspace is rebuilt. +resource "coder_script" "ddev_shutdown" { + agent_id = var.agent_id + display_name = "Stop DDEV Projects" + icon = "/icon/docker.svg" + run_on_stop = true + script = <<-EOT + #!/bin/bash + export PATH="$PATH:/home/linuxbrew/.linuxbrew/bin:/usr/local/bin" + # Wait for Docker socket — it should already be up, but guard against + # race conditions during workspace stop/update. + for i in $(seq 1 10); do + [ -S /var/run/docker.sock ] && break + sleep 1 + done + if [ -S /var/run/docker.sock ]; then + echo "Running ddev poweroff..." + ddev poweroff || true + echo "ddev poweroff complete" + else + echo "Docker socket not available; skipping ddev poweroff" + fi + + DOCKERD_PID=$(pgrep -x dockerd || true) + if [ -n "$DOCKERD_PID" ]; then + echo "Stopping Docker daemon (pid $DOCKERD_PID)..." + sudo kill -TERM "$DOCKERD_PID" 2>/dev/null || true + for i in $(seq 1 30); do + kill -0 "$DOCKERD_PID" 2>/dev/null || break + sleep 1 + done + if kill -0 "$DOCKERD_PID" 2>/dev/null; then + echo "Docker daemon did not stop in time; forcing" + sudo kill -KILL "$DOCKERD_PID" 2>/dev/null || true + else + echo "Docker daemon stopped cleanly" + fi + fi + EOT +} diff --git a/modules/docker-daemon/outputs.tf b/modules/docker-daemon/outputs.tf new file mode 100644 index 0000000..c7d1f96 --- /dev/null +++ b/modules/docker-daemon/outputs.tf @@ -0,0 +1,55 @@ +output "startup_script" { + description = "Bash snippet for the caller to append to its coder_agent startup_script, after registry-mirror configuration and before any command that needs Docker (ddev config global, image pre-pull, etc)." + value = <<-EOT + + # Start Docker Daemon (Sysbox). dockerd runs as a bare background + # process here (no init system inside the container manages it), so + # when something outside Coder restarts this same container in place + # -- e.g. the host's docker-ce package being upgraded, which restarts + # docker.service and, via `restart = unless-stopped`, restarts every + # workspace container -- dockerd is killed without a chance to clean + # up (see modules/docker-daemon for the matching graceful-shutdown + # half of this). That leaves a stale /var/run/docker.pid (and socket + # file) in the container's writable layer, which makes the next + # dockerd refuse to start ("process with PID ... is still running") + # even though nothing is actually running. Clear that leftover state + # before starting, and confirm the daemon actually answers rather than + # trusting the socket file alone. + wait_for_dockerd() { + for i in $(seq 1 30); do + sudo docker info > /dev/null 2>&1 && return 0 + sleep 1 + done + return 1 + } + + if sudo docker info > /dev/null 2>&1; then + echo "Docker Daemon already running." + else + sudo pkill -x dockerd 2>/dev/null || true + sudo rm -f /var/run/docker.pid /var/run/docker.sock + + echo "Starting Docker Daemon..." + sudo dockerd > /tmp/dockerd.log 2>&1 & + if wait_for_dockerd; then + echo "Docker Daemon ready" + else + echo "Docker Daemon not responding after 30s; retrying after cleanup" + sudo pkill -x dockerd 2>/dev/null || true + sleep 2 + sudo rm -f /var/run/docker.pid /var/run/docker.sock + sudo dockerd > /tmp/dockerd.log 2>&1 & + if wait_for_dockerd; then + echo "Docker Daemon ready after retry" + else + echo "Error: Docker Daemon failed to start after cleanup + retry; see /tmp/dockerd.log" + tail -n 40 /tmp/dockerd.log || true + fi + fi + fi + + if [ -S /var/run/docker.sock ]; then + sudo chmod 666 /var/run/docker.sock + fi + EOT +} diff --git a/modules/docker-daemon/variables.tf b/modules/docker-daemon/variables.tf new file mode 100644 index 0000000..f9fb4c9 --- /dev/null +++ b/modules/docker-daemon/variables.tf @@ -0,0 +1,4 @@ +variable "agent_id" { + description = "ID of the coder_agent to attach the Docker daemon shutdown script to" + type = string +} diff --git a/modules/docker-daemon/versions.tf b/modules/docker-daemon/versions.tf new file mode 100644 index 0000000..394c7cb --- /dev/null +++ b/modules/docker-daemon/versions.tf @@ -0,0 +1,7 @@ +terraform { + required_providers { + coder = { + source = "coder/coder" + } + } +}