fix(ci): replace racy staging-box wait with an atomic lock-slot semaphore

.github/workflows/drupal-contrib-integration-test.yml

@@ -37,10 +37,11 @@ env:
 # running plus one pending; every other simultaneous contender gets silently
 # CANCELLED, not queued. Instead, each box-touching job below (including
 # contrib-plain-gh and contrib-issue-fork-gh, which otherwise run in parallel
-# within a single workflow run) calls scripts/ci-wait-for-staging-box.sh right
-# before creating its workspace, which polls the Coder server's actual
-# workspace count and waits its turn -- see that script's header comment for
-# detail.
+# within a single workflow run) calls scripts/ci-acquire-staging-lock.sh right
+# before creating its workspace, which claims one of a fixed number of
+# lock-slot workspaces (an atomic Coder-side compare-and-swap on workspace
+# name) and waits its turn if all slots are held -- see that script's header
+# comment for detail.
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
   cancel-in-progress: true
@@ -67,7 +68,7 @@ jobs:
     if: false
     runs-on: [self-hosted, sysbox]
     # Long enough to cover queued time behind other box-provisioning jobs
-    # (see scripts/ci-wait-for-staging-box.sh) plus this job's own runtime.
+    # (see scripts/ci-acquire-staging-lock.sh) plus this job's own runtime.
     timeout-minutes: 75
     strategy:
       matrix:
@@ -230,7 +231,7 @@ jobs:
     if: false
     runs-on: [self-hosted, sysbox]
     # Long enough to cover queued time behind other box-provisioning jobs
-    # (see scripts/ci-wait-for-staging-box.sh) plus this job's own runtime.
+    # (see scripts/ci-acquire-staging-lock.sh) plus this job's own runtime.
     timeout-minutes: 75
     defaults:
       run:
@@ -410,7 +411,7 @@ jobs:
     if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }}
     runs-on: ubuntu-latest
     # Long enough to cover queued time behind other box-provisioning jobs
-    # (see scripts/ci-wait-for-staging-box.sh) plus this job's own runtime.
+    # (see scripts/ci-acquire-staging-lock.sh) plus this job's own runtime.
     timeout-minutes: 75
     strategy:
       # PRs run a single module (smoke); push-to-main and nightly run the full
@@ -460,8 +461,10 @@ jobs:
       - name: Reap predecessor workspaces (cancelled prior runs of this cell)
         run: ./scripts/ci-reap-family.sh "gc-${{ matrix.project }}-d${{ matrix.drupal_version }}-" "${{ env.WORKSPACE_NAME }}"
 
-      - name: Wait for staging box to be free
-        run: ./scripts/ci-wait-for-staging-box.sh
+      - name: Acquire staging box lock
+        run: ./scripts/ci-acquire-staging-lock.sh
+        env:
+          CI_LOCK_SLOTS: ${{ vars.CI_LOCK_SLOTS }}
 
       - name: Create workspace
         run: |
@@ -547,6 +550,10 @@ jobs:
         if: always()
         run: coder delete ${{ env.WORKSPACE_NAME }} --yes || true
 
+      - name: Release staging box lock
+        if: always()
+        run: ./scripts/ci-release-staging-lock.sh
+
       - name: Archive CI template version
         if: always()
         run: coder templates versions archive drupal-contrib ${{ env.WORKSPACE_NAME }} --yes || true
@@ -556,7 +563,7 @@ jobs:
     if: ${{ vars.CONTRIB_TEST_ISSUE_FORK != '' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner) }}
     runs-on: ubuntu-latest
     # Long enough to cover queued time behind other box-provisioning jobs
-    # (see scripts/ci-wait-for-staging-box.sh) plus this job's own runtime.
+    # (see scripts/ci-acquire-staging-lock.sh) plus this job's own runtime.
     timeout-minutes: 75
     defaults:
       run:
@@ -620,8 +627,10 @@ jobs:
       - name: Reap predecessor workspaces (cancelled prior runs of this cell)
         run: ./scripts/ci-reap-family.sh "gc-fork-" "${{ env.WORKSPACE_NAME }}"
 
-      - name: Wait for staging box to be free
-        run: ./scripts/ci-wait-for-staging-box.sh
+      - name: Acquire staging box lock
+        run: ./scripts/ci-acquire-staging-lock.sh
+        env:
+          CI_LOCK_SLOTS: ${{ vars.CI_LOCK_SLOTS }}
 
       - name: Create workspace
         run: |
@@ -705,6 +714,10 @@ jobs:
         if: always()
         run: coder delete ${{ env.WORKSPACE_NAME }} --yes || true
 
+      - name: Release staging box lock
+        if: always()
+        run: ./scripts/ci-release-staging-lock.sh
+
       - name: Archive CI template version
         if: always()
         run: coder templates versions archive drupal-contrib gc-${{ github.run_number }}-${{ github.run_attempt }} --yes || true

.github/workflows/drupal-integration-test.yml

@@ -43,9 +43,10 @@ env:
 # simultaneous contender gets silently CANCELLED, not queued. Instead, each
 # box-touching job below (including drupal-plain-gh and drupal-issue-fork-gh,
 # which otherwise run in parallel within a single workflow run) calls
-# scripts/ci-wait-for-staging-box.sh right before creating its workspace,
-# which polls the Coder server's actual workspace count and waits its turn --
-# see that script's header comment for detail.
+# scripts/ci-acquire-staging-lock.sh right before creating its workspace,
+# which claims one of a fixed number of lock-slot workspaces (an atomic
+# Coder-side compare-and-swap on workspace name) and waits its turn if all
+# slots are held -- see that script's header comment for detail.
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
   cancel-in-progress: true
@@ -72,7 +73,7 @@ jobs:
     if: false
     runs-on: [self-hosted, sysbox]
     # Long enough to cover queued time behind other box-provisioning jobs
-    # (see scripts/ci-wait-for-staging-box.sh) plus this job's own runtime.
+    # (see scripts/ci-acquire-staging-lock.sh) plus this job's own runtime.
     timeout-minutes: 75
     strategy:
       matrix:
@@ -218,7 +219,7 @@ jobs:
     if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }}
     runs-on: ubuntu-latest
     # Long enough to cover queued time behind other box-provisioning jobs
-    # (see scripts/ci-wait-for-staging-box.sh) plus this job's own runtime.
+    # (see scripts/ci-acquire-staging-lock.sh) plus this job's own runtime.
     timeout-minutes: 75
     strategy:
       # PRs run a single Drupal version (smoke); push-to-main and nightly run the
@@ -266,8 +267,10 @@ jobs:
       - name: Reap predecessor workspaces (cancelled prior runs of this cell)
         run: ./scripts/ci-reap-family.sh "gd-${{ matrix.drupal_version }}-" "${{ env.WORKSPACE_NAME }}"
 
-      - name: Wait for staging box to be free
-        run: ./scripts/ci-wait-for-staging-box.sh
+      - name: Acquire staging box lock
+        run: ./scripts/ci-acquire-staging-lock.sh
+        env:
+          CI_LOCK_SLOTS: ${{ vars.CI_LOCK_SLOTS }}
 
       - name: Create workspace
         run: |
@@ -292,6 +295,10 @@ jobs:
       - *verify-http
       - *delete-workspace
 
+      - name: Release staging box lock
+        if: always()
+        run: ./scripts/ci-release-staging-lock.sh
+
       - name: Archive CI template version
         if: always()
         run: coder templates versions archive drupal-core ${{ env.WORKSPACE_NAME }} --yes || true
@@ -303,7 +310,7 @@ jobs:
     if: false
     runs-on: [self-hosted, sysbox]
     # Long enough to cover queued time behind other box-provisioning jobs
-    # (see scripts/ci-wait-for-staging-box.sh) plus this job's own runtime.
+    # (see scripts/ci-acquire-staging-lock.sh) plus this job's own runtime.
     timeout-minutes: 75
     defaults:
       run:
@@ -467,7 +474,7 @@ jobs:
     if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }}
     runs-on: ubuntu-latest
     # Long enough to cover queued time behind other box-provisioning jobs
-    # (see scripts/ci-wait-for-staging-box.sh) plus this job's own runtime.
+    # (see scripts/ci-acquire-staging-lock.sh) plus this job's own runtime.
     timeout-minutes: 75
     defaults:
       run:
@@ -528,8 +535,10 @@ jobs:
       - name: Reap predecessor workspaces (cancelled prior runs of this cell)
         run: ./scripts/ci-reap-family.sh "gd-fork-" "${{ env.WORKSPACE_NAME }}"
 
-      - name: Wait for staging box to be free
-        run: ./scripts/ci-wait-for-staging-box.sh
+      - name: Acquire staging box lock
+        run: ./scripts/ci-acquire-staging-lock.sh
+        env:
+          CI_LOCK_SLOTS: ${{ vars.CI_LOCK_SLOTS }}
 
       - name: Create workspace
         run: |
@@ -609,6 +618,10 @@ jobs:
         if: always()
         run: coder delete ${{ env.WORKSPACE_NAME }} --yes || true
 
+      - name: Release staging box lock
+        if: always()
+        run: ./scripts/ci-release-staging-lock.sh
+
       - name: Archive CI template version
         if: always()
         run: coder templates versions archive drupal-core gd-${{ github.run_number }}-${{ github.run_attempt }} --yes || true

.github/workflows/integration-test.yml

@@ -71,9 +71,11 @@ on:
 # drupal-contrib-integration-test.yml at nearly the same instant, and GitHub's
 # concurrency groups only keep one job running plus one pending; every other
 # simultaneous contender gets silently CANCELLED, not queued. Instead, each
-# box-touching job below runs scripts/ci-wait-for-staging-box.sh right before
-# creating its workspace, which polls the Coder server's actual workspace
-# count and waits its turn -- see that script's header comment for detail.
+# box-touching job below runs scripts/ci-acquire-staging-lock.sh right before
+# creating its workspace, which claims one of a fixed number of lock-slot
+# workspaces (an atomic Coder-side compare-and-swap on workspace name) and
+# waits its turn if all slots are held -- see that script's header comment
+# for detail.
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
   cancel-in-progress: true
@@ -86,7 +88,7 @@ jobs:
     if: false
     runs-on: [self-hosted, sysbox]
     # Long enough to cover queued time behind other box-provisioning jobs
-    # (see scripts/ci-wait-for-staging-box.sh) plus this job's own runtime.
+    # (see scripts/ci-acquire-staging-lock.sh) plus this job's own runtime.
     timeout-minutes: 75
     strategy:
       matrix:
@@ -302,7 +304,7 @@ jobs:
     if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }}
     runs-on: ubuntu-latest
     # Long enough to cover queued time behind other box-provisioning jobs
-    # (see scripts/ci-wait-for-staging-box.sh) plus this job's own runtime.
+    # (see scripts/ci-acquire-staging-lock.sh) plus this job's own runtime.
     timeout-minutes: 75
     strategy:
       matrix:
@@ -361,8 +363,10 @@ jobs:
       - name: Reap predecessor workspaces (cancelled prior runs of this cell)
         run: ./scripts/ci-reap-family.sh "gh-${{ matrix.ws_name }}-" "${{ env.WORKSPACE_NAME }}"
 
-      - name: Wait for staging box to be free
-        run: ./scripts/ci-wait-for-staging-box.sh
+      - name: Acquire staging box lock
+        run: ./scripts/ci-acquire-staging-lock.sh
+        env:
+          CI_LOCK_SLOTS: ${{ vars.CI_LOCK_SLOTS }}
 
       - name: Create workspace
         if: ${{ matrix.template != 'freeform' }}
@@ -497,6 +501,10 @@ jobs:
         if: always()
         run: coder delete ${{ env.WORKSPACE_NAME }} --yes || true
 
+      - name: Release staging box lock
+        if: always()
+        run: ./scripts/ci-release-staging-lock.sh
+
       - name: Archive CI template version
         if: always()
         run: coder templates versions archive ${{ matrix.template }} ci-gh-${{ env.CI_TAG }} --yes || true

CLAUDE.md

@@ -263,6 +263,7 @@ Key template variables (e.g. in `freeform/template.tf`):
 - Use feature branches for changes
 - **Never use the local `main` branch** — always `git fetch upstream` and base branches on `upstream/main`. Use `upstream/main` for comparisons (e.g. `git diff upstream/main...HEAD`), not local `main`.
 - **Never run `git push`, under any circumstances** — not to `main`, not to a feature branch, not to a fork. Commit locally and hand off to the user (or open a PR only if explicitly asked and only via a mechanism that doesn't require you to push, e.g. `gh pr create` from a branch the user has already pushed) — always let the user push.
+- **Branch naming**: `YYYYMMDD_<username>_<short_description>` (standard DDEV convention, matches `../ddev/CLAUDE.md`). Example: `20250108_rfay_fix_networking`. Create with `git fetch upstream && git checkout -b <branch_name> upstream/main --no-track`.
 - Always use OpenSpec for architectural changes (see AGENTS.md)
 
 ### OpenSpec Integration

Makefile

@@ -6,7 +6,10 @@ VERSION := $(shell cat VERSION 2>/dev/null || echo "1.0.0-beta1")
 DOCKERFILE_DIR := image
 DOCKERFILE := $(DOCKERFILE_DIR)/Dockerfile
 
-# Template directories (name == directory name == Coder template name)
+# Template directories (name == directory name == Coder template name).
+# ci-lock is a separate CI-only mutex template (see ci-lock/template.tf) --
+# deliberately excluded here since it has no shared assets to vendor and no
+# image_version variable; it gets its own validate/push handling below.
 TEMPLATES := drupal-core drupal-contrib freeform
 
 # Host path to the drupal-core seed cache (bind-mounted read-only into workspaces).
@@ -144,6 +147,8 @@ validate: sync-shared ## Validate all Terraform templates (requires terraform in
 		echo "--- Validating $$t ---"; \
 		(cd $$t && terraform init -backend=false -input=false -no-color && terraform validate -no-color) || exit 1; \
 	done
+	@echo "--- Validating ci-lock ---"
+	@(cd ci-lock && terraform init -backend=false -input=false -no-color && terraform validate -no-color) || exit 1
 	@echo "All templates valid."
 
 .PHONY: fmt-check
@@ -207,8 +212,16 @@ push-template-drupal-contrib: sync-shared ## Push drupal-contrib template to Cod
 push-template-freeform: sync-shared ## Push freeform template to Coder
 	$(call push_template,freeform)
 
+.PHONY: push-template-ci-lock
+push-template-ci-lock: ## Push the ci-lock CI-mutex template to Coder (no image, no shared assets -- see ci-lock/template.tf)
+	@echo "Pushing Coder template ci-lock..."
+	coder templates push --directory ci-lock ci-lock --yes --activate=$(ACTIVATE)
+	@echo "Setting template metadata for ci-lock..."
+	coder templates edit ci-lock --yes --display-name "CI Lock (internal)" --description "CI-internal mutex for the staging box. Not a development environment -- do not use."
+	@echo "Template ci-lock push complete"
+
 .PHONY: push-all-templates
-push-all-templates: push-template-drupal-core push-template-drupal-contrib push-template-freeform ## Push all templates to Coder (no image build)
+push-all-templates: push-template-drupal-core push-template-drupal-contrib push-template-freeform push-template-ci-lock ## Push all templates to Coder (no image build)
 	@echo "All templates pushed!"
 
 # --- Deploy targets ---

ci-lock/template.tf

@@ -0,0 +1,27 @@
+# CI-internal mutex, not a development environment.
+#
+# Integration-test CI jobs create real, resource-heavy (Sysbox + Docker-in-Docker)
+# workspaces on a single shared staging Coder box, from both self-hosted runners
+# and GitHub-hosted runners. To bound how many of those run at once without a
+# check-then-act race, jobs claim one of N fixed-name workspaces provisioned from
+# THIS template (ci-slot-1..N) before creating their real workspace, and delete it
+# when done. Coder enforces a unique workspace name per owner, so `coder create
+# ci-slot-<i>` is an atomic compare-and-swap: only one concurrent caller can win
+# for a given <i>. See scripts/ci-acquire-staging-lock.sh /
+# scripts/ci-release-staging-lock.sh for the protocol, and
+# openspec/changes/add-ci-staging-lock/design.md for why this approach was chosen
+# over a git-ref-based lock.
+#
+# Deliberately has no coder_agent: a lock slot is never connected to, only
+# created and deleted, so it provisions and tears down in a second or two.
+
+terraform {
+  required_providers {
+    coder = {
+      source  = "coder/coder"
+      version = ">= 2.13"
+    }
+  }
+}
+
+resource "terraform_data" "lock" {}

docs/admin/server-setup.md

@@ -1243,6 +1243,17 @@ coder tokens create --user ci-bot --lifetime 8760h
 
 Store the token in 1Password at `op://test-secrets/TEST_CODER_SESSION_TOKEN/credential`.
 
+#### 3. Push the `ci-lock` template
+
+The integration-test workflows serialize access to this box through `scripts/ci-acquire-staging-lock.sh`, which claims one of a fixed set of lock-slot workspaces provisioned from the `ci-lock` template (see "Tuning how many CI workspaces can run at once" below). That template is CI-only plumbing — no workflow pushes it automatically — so it needs to exist before the first CI run. `make push-all-templates` (already the standard way to push templates to an environment) covers it along with the rest; log in as a template-admin (e.g. `ci-bot`) first:
+
+```bash
+coder login <staging-coder-url>
+make push-all-templates
+```
+
+Without this, every `ci-acquire-staging-lock.sh` invocation fails fast with "no 'ci-lock' template found".
+
 ### GitHub repository configuration
 
 Go to **GitHub → Settings → Secrets and variables → Actions** and add:
@@ -1259,6 +1270,19 @@ Go to **GitHub → Settings → Secrets and variables → Actions** and add:
 |--------------------------|-----------------------------------------|
 | `TEST_CODER_URL`         | `https://staging-coder.ddev.com`        |
 | `DRUPAL_TEST_ISSUE_FORK` | A drupal.org issue number (see below)   |
+| `CI_LOCK_SLOTS`          | Max concurrent CI workspaces on staging (see below) |
+
+### Tuning how many CI workspaces can run at once (`CI_LOCK_SLOTS`)
+
+`integration-test.yml`, `drupal-integration-test.yml`, and `drupal-contrib-integration-test.yml` all create real, resource-heavy (Sysbox + Docker-in-Docker) workspaces on the single shared staging box. Before creating one, each job runs `scripts/ci-acquire-staging-lock.sh`, which claims one of a fixed number of lock-slot workspaces (`ci-slot-1`..`ci-slot-N`) and waits if all are held — see that script's header comment and `openspec/changes/add-ci-staging-lock/design.md` for the full mechanism.
+
+`N` is controlled entirely by the `CI_LOCK_SLOTS` repository variable — no code change needed to retune it:
+
+- **Unset**: defaults to `2`.
+- **Raise it** if staging has spare CPU/RAM headroom and jobs are spending a lot of time queued behind `ci-acquire-staging-lock.sh` waiting for a slot.
+- **Lower it** if jobs are failing with agent-connection errors (e.g. "Agent doesn't exist with that id") that trace back to the box being oversubscribed — each workspace's default request is 4 CPU / 8GB, so pick `N` with that against the box's actual cores/RAM in mind.
+
+Change it in **GitHub → Settings → Secrets and variables → Actions → Variables**; it takes effect on the next workflow run, no restart or redeploy required.
 
 ### Choosing a test issue for `DRUPAL_TEST_ISSUE_FORK`