fix(startup): self-heal nested Docker daemon after in-place container restart

Makefile

@@ -119,8 +119,17 @@ sync-vscode-extensions: ## Vendor shared/vscode-extensions.tf into each template
 	done
 	@echo "Synced shared/vscode-extensions.tf into: $(TEMPLATES)"
 
+.PHONY: sync-docker-daemon-module
+sync-docker-daemon-module: ## Vendor modules/docker-daemon into each template dir (same reason as sync-claude-module: push only bundles the given --directory)
+	@for t in $(TEMPLATES); do \
+		rm -rf $$t/modules/docker-daemon; \
+		mkdir -p $$t/modules; \
+		cp -r modules/docker-daemon $$t/modules/docker-daemon; \
+	done
+	@echo "Synced modules/docker-daemon into: $(TEMPLATES)"
+
 .PHONY: sync-shared
-sync-shared: sync-claude-module sync-vscode-extensions ## Vendor all shared Terraform assets (modules + vscode-extensions.tf) into each template dir
+sync-shared: sync-claude-module sync-vscode-extensions sync-docker-daemon-module ## Vendor all shared Terraform assets (modules + vscode-extensions.tf) into each template dir
 
 .PHONY: validate
 validate: sync-shared ## Validate all Terraform templates (requires terraform in PATH)

drupal-contrib/modules/docker-daemon/main.tf

@@ -0,0 +1,49 @@
+# Stops the nested Docker daemon (Sysbox) cleanly on workspace stop. It runs
+# as a bare background process in the caller's startup script (see
+# outputs.tf) -- no init system inside the container manages it -- so
+# nothing else sends it a graceful shutdown before the container itself is
+# torn down. That matters beyond a normal workspace stop: when something
+# outside Coder restarts the container in place (e.g. the host's docker-ce
+# package being upgraded, which restarts docker.service and, via `restart =
+# unless-stopped`, restarts every workspace container), an unclean kill here
+# leaves a stale /var/run/docker.pid behind that blocks the next dockerd
+# from starting until the workspace is rebuilt.
+resource "coder_script" "ddev_shutdown" {
+  agent_id     = var.agent_id
+  display_name = "Stop DDEV Projects"
+  icon         = "/icon/docker.svg"
+  run_on_stop  = true
+  script       = <<-EOT
+    #!/bin/bash
+    export PATH="$PATH:/home/linuxbrew/.linuxbrew/bin:/usr/local/bin"
+    # Wait for Docker socket — it should already be up, but guard against
+    # race conditions during workspace stop/update.
+    for i in $(seq 1 10); do
+      [ -S /var/run/docker.sock ] && break
+      sleep 1
+    done
+    if [ -S /var/run/docker.sock ]; then
+      echo "Running ddev poweroff..."
+      ddev poweroff || true
+      echo "ddev poweroff complete"
+    else
+      echo "Docker socket not available; skipping ddev poweroff"
+    fi
+
+    DOCKERD_PID=$(pgrep -x dockerd || true)
+    if [ -n "$DOCKERD_PID" ]; then
+      echo "Stopping Docker daemon (pid $DOCKERD_PID)..."
+      sudo kill -TERM "$DOCKERD_PID" 2>/dev/null || true
+      for i in $(seq 1 30); do
+        kill -0 "$DOCKERD_PID" 2>/dev/null || break
+        sleep 1
+      done
+      if kill -0 "$DOCKERD_PID" 2>/dev/null; then
+        echo "Docker daemon did not stop in time; forcing"
+        sudo kill -KILL "$DOCKERD_PID" 2>/dev/null || true
+      else
+        echo "Docker daemon stopped cleanly"
+      fi
+    fi
+  EOT
+}

drupal-contrib/modules/docker-daemon/outputs.tf

@@ -0,0 +1,55 @@
+output "startup_script" {
+  description = "Bash snippet for the caller to append to its coder_agent startup_script, after registry-mirror configuration and before any command that needs Docker (ddev config global, image pre-pull, etc)."
+  value       = <<-EOT
+
+    # Start Docker Daemon (Sysbox). dockerd runs as a bare background
+    # process here (no init system inside the container manages it), so
+    # when something outside Coder restarts this same container in place
+    # -- e.g. the host's docker-ce package being upgraded, which restarts
+    # docker.service and, via `restart = unless-stopped`, restarts every
+    # workspace container -- dockerd is killed without a chance to clean
+    # up (see modules/docker-daemon for the matching graceful-shutdown
+    # half of this). That leaves a stale /var/run/docker.pid (and socket
+    # file) in the container's writable layer, which makes the next
+    # dockerd refuse to start ("process with PID ... is still running")
+    # even though nothing is actually running. Clear that leftover state
+    # before starting, and confirm the daemon actually answers rather than
+    # trusting the socket file alone.
+    wait_for_dockerd() {
+      for i in $(seq 1 30); do
+        sudo docker info > /dev/null 2>&1 && return 0
+        sleep 1
+      done
+      return 1
+    }
+
+    if sudo docker info > /dev/null 2>&1; then
+      echo "Docker Daemon already running."
+    else
+      sudo pkill -x dockerd 2>/dev/null || true
+      sudo rm -f /var/run/docker.pid /var/run/docker.sock
+
+      echo "Starting Docker Daemon..."
+      sudo dockerd > /tmp/dockerd.log 2>&1 &
+      if wait_for_dockerd; then
+        echo "Docker Daemon ready"
+      else
+        echo "Docker Daemon not responding after 30s; retrying after cleanup"
+        sudo pkill -x dockerd 2>/dev/null || true
+        sleep 2
+        sudo rm -f /var/run/docker.pid /var/run/docker.sock
+        sudo dockerd > /tmp/dockerd.log 2>&1 &
+        if wait_for_dockerd; then
+          echo "Docker Daemon ready after retry"
+        else
+          echo "Error: Docker Daemon failed to start after cleanup + retry; see /tmp/dockerd.log"
+          tail -n 40 /tmp/dockerd.log || true
+        fi
+      fi
+    fi
+
+    if [ -S /var/run/docker.sock ]; then
+      sudo chmod 666 /var/run/docker.sock
+    fi
+  EOT
+}

drupal-contrib/modules/docker-daemon/variables.tf

@@ -0,0 +1,4 @@
+variable "agent_id" {
+  description = "ID of the coder_agent to attach the Docker daemon shutdown script to"
+  type        = string
+}

drupal-contrib/modules/docker-daemon/versions.tf

@@ -0,0 +1,7 @@
+terraform {
+  required_providers {
+    coder = {
+      source = "coder/coder"
+    }
+  }
+}

drupal-contrib/template.tf

@@ -407,27 +407,7 @@ resource "coder_agent" "main" {
 EOF
     fi
 
-    if ! pgrep -x "dockerd" > /dev/null; then
-      echo "Starting Docker Daemon..."
-      sudo dockerd > /tmp/dockerd.log 2>&1 &
-
-      echo "Waiting for Docker Socket..."
-      for i in $(seq 1 30); do
-        if [ -S /var/run/docker.sock ]; then
-          echo "Docker Socket found!"
-          break
-        fi
-        sleep 1
-      done
-
-      if [ -S /var/run/docker.sock ]; then
-        sudo chmod 666 /var/run/docker.sock
-      else
-        echo "Error: Docker Socket not found after 30s!"
-      fi
-    else
-      echo "Docker Daemon already running."
-    fi
+    ${module.docker_daemon.startup_script}
 
     mkdir -p ~/.ddev
     echo "Configuring DDEV to omit ddev-router..."
@@ -1160,26 +1140,9 @@ resource "coder_app" "mailpit" {
   }
 }
 
-resource "coder_script" "ddev_shutdown" {
-  agent_id     = coder_agent.main.id
-  display_name = "Stop DDEV Projects"
-  icon         = "/icon/docker.svg"
-  run_on_stop  = true
-  script       = <<-EOT
-    #!/bin/bash
-    export PATH="$PATH:/home/linuxbrew/.linuxbrew/bin:/usr/local/bin"
-    for i in $(seq 1 10); do
-      [ -S /var/run/docker.sock ] && break
-      sleep 1
-    done
-    if [ ! -S /var/run/docker.sock ]; then
-      echo "Docker socket not available; skipping ddev poweroff"
-      exit 0
-    fi
-    echo "Running ddev poweroff..."
-    ddev poweroff || true
-    echo "ddev poweroff complete"
-  EOT
+module "docker_daemon" {
+  source   = "./modules/docker-daemon"
+  agent_id = coder_agent.main.id
 }
 
 resource "docker_container" "workspace" {

drupal-core/modules/docker-daemon/main.tf

@@ -0,0 +1,49 @@
+# Stops the nested Docker daemon (Sysbox) cleanly on workspace stop. It runs
+# as a bare background process in the caller's startup script (see
+# outputs.tf) -- no init system inside the container manages it -- so
+# nothing else sends it a graceful shutdown before the container itself is
+# torn down. That matters beyond a normal workspace stop: when something
+# outside Coder restarts the container in place (e.g. the host's docker-ce
+# package being upgraded, which restarts docker.service and, via `restart =
+# unless-stopped`, restarts every workspace container), an unclean kill here
+# leaves a stale /var/run/docker.pid behind that blocks the next dockerd
+# from starting until the workspace is rebuilt.
+resource "coder_script" "ddev_shutdown" {
+  agent_id     = var.agent_id
+  display_name = "Stop DDEV Projects"
+  icon         = "/icon/docker.svg"
+  run_on_stop  = true
+  script       = <<-EOT
+    #!/bin/bash
+    export PATH="$PATH:/home/linuxbrew/.linuxbrew/bin:/usr/local/bin"
+    # Wait for Docker socket — it should already be up, but guard against
+    # race conditions during workspace stop/update.
+    for i in $(seq 1 10); do
+      [ -S /var/run/docker.sock ] && break
+      sleep 1
+    done
+    if [ -S /var/run/docker.sock ]; then
+      echo "Running ddev poweroff..."
+      ddev poweroff || true
+      echo "ddev poweroff complete"
+    else
+      echo "Docker socket not available; skipping ddev poweroff"
+    fi
+
+    DOCKERD_PID=$(pgrep -x dockerd || true)
+    if [ -n "$DOCKERD_PID" ]; then
+      echo "Stopping Docker daemon (pid $DOCKERD_PID)..."
+      sudo kill -TERM "$DOCKERD_PID" 2>/dev/null || true
+      for i in $(seq 1 30); do
+        kill -0 "$DOCKERD_PID" 2>/dev/null || break
+        sleep 1
+      done
+      if kill -0 "$DOCKERD_PID" 2>/dev/null; then
+        echo "Docker daemon did not stop in time; forcing"
+        sudo kill -KILL "$DOCKERD_PID" 2>/dev/null || true
+      else
+        echo "Docker daemon stopped cleanly"
+      fi
+    fi
+  EOT
+}

drupal-core/modules/docker-daemon/outputs.tf

@@ -0,0 +1,55 @@
+output "startup_script" {
+  description = "Bash snippet for the caller to append to its coder_agent startup_script, after registry-mirror configuration and before any command that needs Docker (ddev config global, image pre-pull, etc)."
+  value       = <<-EOT
+
+    # Start Docker Daemon (Sysbox). dockerd runs as a bare background
+    # process here (no init system inside the container manages it), so
+    # when something outside Coder restarts this same container in place
+    # -- e.g. the host's docker-ce package being upgraded, which restarts
+    # docker.service and, via `restart = unless-stopped`, restarts every
+    # workspace container -- dockerd is killed without a chance to clean
+    # up (see modules/docker-daemon for the matching graceful-shutdown
+    # half of this). That leaves a stale /var/run/docker.pid (and socket
+    # file) in the container's writable layer, which makes the next
+    # dockerd refuse to start ("process with PID ... is still running")
+    # even though nothing is actually running. Clear that leftover state
+    # before starting, and confirm the daemon actually answers rather than
+    # trusting the socket file alone.
+    wait_for_dockerd() {
+      for i in $(seq 1 30); do
+        sudo docker info > /dev/null 2>&1 && return 0
+        sleep 1
+      done
+      return 1
+    }
+
+    if sudo docker info > /dev/null 2>&1; then
+      echo "Docker Daemon already running."
+    else
+      sudo pkill -x dockerd 2>/dev/null || true
+      sudo rm -f /var/run/docker.pid /var/run/docker.sock
+
+      echo "Starting Docker Daemon..."
+      sudo dockerd > /tmp/dockerd.log 2>&1 &
+      if wait_for_dockerd; then
+        echo "Docker Daemon ready"
+      else
+        echo "Docker Daemon not responding after 30s; retrying after cleanup"
+        sudo pkill -x dockerd 2>/dev/null || true
+        sleep 2
+        sudo rm -f /var/run/docker.pid /var/run/docker.sock
+        sudo dockerd > /tmp/dockerd.log 2>&1 &
+        if wait_for_dockerd; then
+          echo "Docker Daemon ready after retry"
+        else
+          echo "Error: Docker Daemon failed to start after cleanup + retry; see /tmp/dockerd.log"
+          tail -n 40 /tmp/dockerd.log || true
+        fi
+      fi
+    fi
+
+    if [ -S /var/run/docker.sock ]; then
+      sudo chmod 666 /var/run/docker.sock
+    fi
+  EOT
+}

drupal-core/modules/docker-daemon/variables.tf

@@ -0,0 +1,4 @@
+variable "agent_id" {
+  description = "ID of the coder_agent to attach the Docker daemon shutdown script to"
+  type        = string
+}

drupal-core/modules/docker-daemon/versions.tf

@@ -0,0 +1,7 @@
+terraform {
+  required_providers {
+    coder = {
+      source = "coder/coder"
+    }
+  }
+}