From 50fb65146333b379eb24d09c708f5c4320889e6f Mon Sep 17 00:00:00 2001
From: Waleed Latif <walif6@gmail.com>
Date: Wed, 15 Jul 2026 19:16:37 -0700
Subject: [PATCH 1/2] improvement(ci): decouple image builds from the test gate
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- build-amd64 now starts immediately and pushes only sha tags (ECR :sha,
  GHCR :sha-amd64). The EventBridge deploy triggers filter on exactly the
  latest/staging/dev ECR tags, so nothing deploys from these pushes.
- New promote-images job retags sha -> latest/staging (and GHCR
  latest-amd64/version-amd64) via buildx imagetools once tests and
  migrations pass — a seconds-long manifest copy instead of rebuilding
  after the gate. Cuts push-to-deploy from ~13.5 to ~7 minutes.
- Split the Next.js production build out of test-build into a parallel
  Build App job with its own sticky-disk keys, cutting PR feedback from
  ~5.5 to ~3.5 minutes.
- create-ghcr-manifests and process-docs now gate on promote-images.
---
 .github/workflows/ci.yml         | 125 ++++++++++++++++++++++++-------
 .github/workflows/test-build.yml |  76 ++++++++++++++-----
 2 files changed, 158 insertions(+), 43 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 25d8348d8c4..1ad6e249b80 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -46,9 +46,10 @@ jobs:
             echo "ℹ️ Not a release commit"
           fi
 
-  # Run database migrations before images are pushed: the ECR push triggers
-  # CodePipeline, so migrating first guarantees the schema is in place before
-  # the new app version deploys (replaces the removed ECS migration sidecar)
+  # Run database migrations before images are promoted: the ECR latest/staging
+  # tag push triggers CodePipeline, so migrating first guarantees the schema is
+  # in place before the new app version deploys (replaces the removed ECS
+  # migration sidecar)
   migrate:
     name: Migrate DB
     needs: [test-build]
@@ -173,10 +174,13 @@ jobs:
           fi
           bunx trigger.dev@4.4.3 deploy --env preview --branch dev-sim
 
-  # Main/staging: build AMD64 images and push to ECR + GHCR
+  # Main/staging: build AMD64 images and push sha-tagged images to ECR + GHCR.
+  # Runs in parallel with tests — only sha tags are pushed here, and the
+  # CodePipeline EventBridge triggers filter on exactly the latest/staging/dev
+  # ECR tags, so nothing deploys until promote-images retags after the gate.
   build-amd64:
     name: Build AMD64
-    needs: [test-build, detect-version, migrate]
+    needs: [detect-version]
     if: >-
       github.event_name == 'push' &&
       (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging')
@@ -238,6 +242,9 @@ jobs:
         env:
           ECR_REPO: ${{ matrix.ecr_repo_secret == 'ECR_APP' && secrets.ECR_APP || matrix.ecr_repo_secret == 'ECR_MIGRATIONS' && secrets.ECR_MIGRATIONS || matrix.ecr_repo_secret == 'ECR_REALTIME' && secrets.ECR_REALTIME || matrix.ecr_repo_secret == 'ECR_PII' && secrets.ECR_PII || '' }}
 
+      # Only sha tags here — the deploy tags (latest/staging) and the GHCR
+      # latest-amd64/version tags are applied by promote-images after tests
+      # and migrations pass.
       - name: Generate tags
         id: meta
         run: |
@@ -245,26 +252,10 @@ jobs:
           ECR_REPO="${{ steps.ecr-repo.outputs.name }}"
           GHCR_IMAGE="${{ matrix.ghcr_image }}"
 
-          if [ "${{ github.ref }}" = "refs/heads/main" ]; then
-            ECR_TAG="latest"
-          else
-            ECR_TAG="staging"
-          fi
-          ECR_IMAGE="${ECR_REGISTRY}/${ECR_REPO}:${ECR_TAG}"
-
-          TAGS="${ECR_IMAGE}"
+          TAGS="${ECR_REGISTRY}/${ECR_REPO}:${{ github.sha }}"
 
           if [ "${{ github.ref }}" = "refs/heads/main" ] && [ -n "$GHCR_IMAGE" ]; then
-            GHCR_AMD64="${GHCR_IMAGE}:latest-amd64"
-            GHCR_SHA="${GHCR_IMAGE}:${{ github.sha }}-amd64"
-            TAGS="${TAGS},$GHCR_AMD64,$GHCR_SHA"
-
-            if [ "${{ needs.detect-version.outputs.is_release }}" = "true" ]; then
-              VERSION="${{ needs.detect-version.outputs.version }}"
-              GHCR_VERSION="${GHCR_IMAGE}:${VERSION}-amd64"
-              TAGS="${TAGS},$GHCR_VERSION"
-              echo "📦 Adding version tag: ${VERSION}-amd64"
-            fi
+            TAGS="${TAGS},${GHCR_IMAGE}:${{ github.sha }}-amd64"
           fi
 
           echo "tags=${TAGS}" >> $GITHUB_OUTPUT
@@ -280,6 +271,88 @@ jobs:
           provenance: false
           sbom: false
 
+  # Promote the sha-tagged images to the deploy tags once tests and migrations
+  # pass. Pushing the ECR latest/staging tag is what triggers CodePipeline, so
+  # this seconds-long manifest retag is the deploy gate — the image builds
+  # themselves run in parallel with the tests.
+  promote-images:
+    name: Promote Images
+    needs: [detect-version, migrate, build-amd64]
+    if: >-
+      github.event_name == 'push' &&
+      (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging')
+    runs-on: blacksmith-2vcpu-ubuntu-2404
+    permissions:
+      contents: read
+      packages: write
+      id-token: write
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - ghcr_image: ghcr.io/simstudioai/simstudio
+            ecr_repo_secret: ECR_APP
+          - ghcr_image: ghcr.io/simstudioai/migrations
+            ecr_repo_secret: ECR_MIGRATIONS
+          - ghcr_image: ghcr.io/simstudioai/realtime
+            ecr_repo_secret: ECR_REALTIME
+          - ghcr_image: ghcr.io/simstudioai/pii
+            ecr_repo_secret: ECR_PII
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6
+        with:
+          role-to-assume: ${{ github.ref == 'refs/heads/main' && secrets.AWS_ROLE_TO_ASSUME || secrets.STAGING_AWS_ROLE_TO_ASSUME }}
+          aws-region: ${{ github.ref == 'refs/heads/main' && secrets.AWS_REGION || secrets.STAGING_AWS_REGION }}
+
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@d539f0932e70871a027e9d5a9d8fc38589180a64 # v2
+
+      - name: Login to GHCR
+        if: github.ref == 'refs/heads/main'
+        uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Resolve ECR repo name
+        id: ecr-repo
+        run: echo "name=$ECR_REPO" >> $GITHUB_OUTPUT
+        env:
+          ECR_REPO: ${{ matrix.ecr_repo_secret == 'ECR_APP' && secrets.ECR_APP || matrix.ecr_repo_secret == 'ECR_MIGRATIONS' && secrets.ECR_MIGRATIONS || matrix.ecr_repo_secret == 'ECR_REALTIME' && secrets.ECR_REALTIME || matrix.ecr_repo_secret == 'ECR_PII' && secrets.ECR_PII || '' }}
+
+      - name: Promote images to deploy tags
+        run: |
+          ECR_IMAGE="${{ steps.login-ecr.outputs.registry }}/${{ steps.ecr-repo.outputs.name }}"
+          GHCR_IMAGE="${{ matrix.ghcr_image }}"
+
+          if [ "${{ github.ref }}" = "refs/heads/main" ]; then
+            ECR_TAG="latest"
+          else
+            ECR_TAG="staging"
+          fi
+
+          echo "🚀 Promoting ${ECR_IMAGE}:${{ github.sha }} to ${ECR_TAG}"
+          docker buildx imagetools create \
+            -t "${ECR_IMAGE}:${ECR_TAG}" \
+            "${ECR_IMAGE}:${{ github.sha }}"
+
+          if [ "${{ github.ref }}" = "refs/heads/main" ] && [ -n "$GHCR_IMAGE" ]; then
+            docker buildx imagetools create \
+              -t "${GHCR_IMAGE}:latest-amd64" \
+              "${GHCR_IMAGE}:${{ github.sha }}-amd64"
+
+            if [ "${{ needs.detect-version.outputs.is_release }}" = "true" ]; then
+              VERSION="${{ needs.detect-version.outputs.version }}"
+              echo "📦 Adding version tag: ${VERSION}-amd64"
+              docker buildx imagetools create \
+                -t "${GHCR_IMAGE}:${VERSION}-amd64" \
+                "${GHCR_IMAGE}:${{ github.sha }}-amd64"
+            fi
+          fi
+
   # Build ARM64 images for GHCR (main branch only, runs in parallel)
   build-ghcr-arm64:
     name: Build ARM64 (GHCR Only)
@@ -346,7 +419,7 @@ jobs:
   create-ghcr-manifests:
     name: Create GHCR Manifests
     runs-on: blacksmith-2vcpu-ubuntu-2404
-    needs: [build-amd64, build-ghcr-arm64, detect-version]
+    needs: [promote-images, build-ghcr-arm64, detect-version]
     if: github.event_name == 'push' && github.ref == 'refs/heads/main'
     permissions:
       packages: write
@@ -412,10 +485,10 @@ jobs:
               - 'apps/sim/scripts/process-docs.ts'
               - 'apps/sim/lib/chunkers/**'
 
-  # Process docs embeddings (only when docs change, after ECR images are pushed)
+  # Process docs embeddings (only when docs change, after images are promoted)
   process-docs:
     name: Process Docs
-    needs: [build-amd64, check-docs-changes]
+    needs: [promote-images, check-docs-changes]
     if: needs.check-docs-changes.outputs.docs_changed == 'true'
     uses: ./.github/workflows/docs-embeddings.yml
     secrets: inherit
diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml
index 20c2239714c..239d0586d7f 100644
--- a/.github/workflows/test-build.yml
+++ b/.github/workflows/test-build.yml
@@ -9,7 +9,7 @@ permissions:
 
 jobs:
   test-build:
-    name: Test and Build
+    name: Lint and Test
     runs-on: blacksmith-8vcpu-ubuntu-2404
 
     steps:
@@ -44,14 +44,6 @@ jobs:
           key: ${{ github.repository }}-turbo-cache
           path: ./.turbo
 
-      - name: Restore Next.js build cache
-        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
-        with:
-          path: ./apps/sim/.next/cache
-          key: ${{ runner.os }}-nextjs-${{ hashFiles('bun.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-nextjs-
-
       - name: Install dependencies
         run: bun install --frozen-lockfile
 
@@ -175,6 +167,63 @@ jobs:
           fi
           echo "✅ Schema and migrations are in sync"
 
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5
+        with:
+          directory: ./apps/sim/coverage
+          fail_ci_if_error: false
+          verbose: true
+
+  # Next.js production build, in parallel with lint + tests. Uses its own
+  # sticky-disk keys — a sticky disk key can only be mounted by one job at a
+  # time, so sharing keys with test-build would serialize the two jobs.
+  build:
+    name: Build App
+    runs-on: blacksmith-8vcpu-ubuntu-2404
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
+
+      - name: Setup Bun
+        uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
+        with:
+          bun-version: 1.3.13
+
+      - name: Setup Node
+        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
+        with:
+          node-version: latest
+
+      - name: Mount Bun cache (Sticky Disk)
+        uses: useblacksmith/stickydisk@4c034ba57b706cf0e3b4b0ce098c2a3b1071580c # v1
+        with:
+          key: ${{ github.repository }}-bun-cache-build
+          path: ~/.bun/install/cache
+
+      - name: Mount node_modules (Sticky Disk)
+        uses: useblacksmith/stickydisk@4c034ba57b706cf0e3b4b0ce098c2a3b1071580c # v1
+        with:
+          key: ${{ github.repository }}-node-modules-build
+          path: ./node_modules
+
+      - name: Mount Turbo cache (Sticky Disk)
+        uses: useblacksmith/stickydisk@4c034ba57b706cf0e3b4b0ce098c2a3b1071580c # v1
+        with:
+          key: ${{ github.repository }}-turbo-cache-build
+          path: ./.turbo
+
+      - name: Restore Next.js build cache
+        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
+        with:
+          path: ./apps/sim/.next/cache
+          key: ${{ runner.os }}-nextjs-${{ hashFiles('bun.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-nextjs-
+
+      - name: Install dependencies
+        run: bun install --frozen-lockfile
+
       - name: Build application
         env:
           NODE_OPTIONS: '--no-warnings --max-old-space-size=8192'
@@ -186,11 +235,4 @@ jobs:
           AWS_REGION: 'us-west-2'
           ENCRYPTION_KEY: '7cf672e460e430c1fba707575c2b0e2ad5a99dddf9b7b7e3b5646e630861db1c' # dummy key for CI only
           TURBO_CACHE_DIR: .turbo
-        run: bunx turbo run build --filter=sim
-
-      - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5
-        with:
-          directory: ./apps/sim/coverage
-          fail_ci_if_error: false
-          verbose: true
\ No newline at end of file
+        run: bunx turbo run build --filter=sim
\ No newline at end of file

From f457ba055c7eefb66a1942cb3834831a30118b32 Mon Sep 17 00:00:00 2001
From: Waleed Latif <walif6@gmail.com>
Date: Wed, 15 Jul 2026 19:34:09 -0700
Subject: [PATCH 2/2] =?UTF-8?q?improvement(ci):=20harden=20promotion=20?=
 =?UTF-8?q?=E2=80=94=20atomic=20retag,=20stale-run=20guards,=20gate=20all?=
 =?UTF-8?q?=20mutable=20GHCR=20tags?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Review follow-ups:
- promote-images is a single job (not a matrix): verifies all four :sha
  manifests exist before moving any deploy tag, so a missing image can't
  cause a partial mixed-version deploy
- stale-run guard on promote-images and create-ghcr-manifests: re-running
  an old run no longer retags latest/staging back to stale code (a
  one-click prod rollback); superseded runs skip mutable tags with a
  warning while immutable sha/version tags still publish
- ARM64 build now pushes only the immutable :sha-arm64 tag; latest-arm64
  and version tags moved behind the gate into create-ghcr-manifests
  (closes the pre-existing hole where a failing run moved latest-arm64)
- dropped dead detect-version needs from both build jobs
- sticky-disk comment corrected (clone + last-writer-wins, not exclusive
  mounts); build job shares warm bun/node_modules disks, keeps its own
  turbo cache key so test/build entries don't evict each other
---
 .github/workflows/ci.yml         | 181 +++++++++++++++----------------
 .github/workflows/test-build.yml |  14 ++-
 2 files changed, 96 insertions(+), 99 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 1ad6e249b80..88487ace2df 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -175,12 +175,12 @@ jobs:
           bunx trigger.dev@4.4.3 deploy --env preview --branch dev-sim
 
   # Main/staging: build AMD64 images and push sha-tagged images to ECR + GHCR.
-  # Runs in parallel with tests — only sha tags are pushed here, and the
-  # CodePipeline EventBridge triggers filter on exactly the latest/staging/dev
-  # ECR tags, so nothing deploys until promote-images retags after the gate.
+  # Runs in parallel with tests — only immutable sha tags are pushed here, and
+  # the CodePipeline EventBridge triggers filter on exactly the
+  # latest/staging/dev ECR tags, so nothing deploys and no mutable tag moves
+  # until promote-images / create-ghcr-manifests retag after the gate.
   build-amd64:
     name: Build AMD64
-    needs: [detect-version]
     if: >-
       github.event_name == 'push' &&
       (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging')
@@ -242,9 +242,9 @@ jobs:
         env:
           ECR_REPO: ${{ matrix.ecr_repo_secret == 'ECR_APP' && secrets.ECR_APP || matrix.ecr_repo_secret == 'ECR_MIGRATIONS' && secrets.ECR_MIGRATIONS || matrix.ecr_repo_secret == 'ECR_REALTIME' && secrets.ECR_REALTIME || matrix.ecr_repo_secret == 'ECR_PII' && secrets.ECR_PII || '' }}
 
-      # Only sha tags here — the deploy tags (latest/staging) and the GHCR
-      # latest-amd64/version tags are applied by promote-images after tests
-      # and migrations pass.
+      # Only sha tags here — the ECR deploy tags (latest/staging) are applied
+      # by promote-images and the GHCR latest-amd64/version tags by
+      # create-ghcr-manifests, both after tests and migrations pass.
       - name: Generate tags
         id: meta
         run: |
@@ -271,33 +271,22 @@ jobs:
           provenance: false
           sbom: false
 
-  # Promote the sha-tagged images to the deploy tags once tests and migrations
-  # pass. Pushing the ECR latest/staging tag is what triggers CodePipeline, so
-  # this seconds-long manifest retag is the deploy gate — the image builds
-  # themselves run in parallel with the tests.
+  # Promote the sha-tagged ECR images to the deploy tags once tests and
+  # migrations pass. Pushing the ECR latest/staging tag is what triggers
+  # CodePipeline, so this seconds-long manifest retag is the deploy gate —
+  # the image builds themselves run in parallel with the tests. A single job
+  # (not a matrix) so all four sha manifests are verified before any tag
+  # moves; a missing image can't produce a partial mixed-version deploy.
   promote-images:
     name: Promote Images
-    needs: [detect-version, migrate, build-amd64]
+    needs: [migrate, build-amd64]
     if: >-
       github.event_name == 'push' &&
       (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging')
     runs-on: blacksmith-2vcpu-ubuntu-2404
     permissions:
       contents: read
-      packages: write
       id-token: write
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - ghcr_image: ghcr.io/simstudioai/simstudio
-            ecr_repo_secret: ECR_APP
-          - ghcr_image: ghcr.io/simstudioai/migrations
-            ecr_repo_secret: ECR_MIGRATIONS
-          - ghcr_image: ghcr.io/simstudioai/realtime
-            ecr_repo_secret: ECR_REALTIME
-          - ghcr_image: ghcr.io/simstudioai/pii
-            ecr_repo_secret: ECR_PII
     steps:
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6
@@ -309,24 +298,31 @@ jobs:
         id: login-ecr
         uses: aws-actions/amazon-ecr-login@d539f0932e70871a027e9d5a9d8fc38589180a64 # v2
 
-      - name: Login to GHCR
-        if: github.ref == 'refs/heads/main'
-        uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Resolve ECR repo name
-        id: ecr-repo
-        run: echo "name=$ECR_REPO" >> $GITHUB_OUTPUT
-        env:
-          ECR_REPO: ${{ matrix.ecr_repo_secret == 'ECR_APP' && secrets.ECR_APP || matrix.ecr_repo_secret == 'ECR_MIGRATIONS' && secrets.ECR_MIGRATIONS || matrix.ecr_repo_secret == 'ECR_REALTIME' && secrets.ECR_REALTIME || matrix.ecr_repo_secret == 'ECR_PII' && secrets.ECR_PII || '' }}
+      # Re-running this job from an old run would retag the deploy tags back
+      # to that run's commit and immediately trigger a production deploy of
+      # stale code. Only promote while this commit is still the branch head;
+      # when superseded, skip cleanly — the newer run's promotion covers it.
+      - name: Guard against stale promotion
+        id: guard
+        run: |
+          HEAD_SHA="$(git ls-remote "https://github.com/${{ github.repository }}.git" "refs/heads/${GITHUB_REF_NAME}" | cut -f1)"
+          if [ "$HEAD_SHA" != "${{ github.sha }}" ]; then
+            echo "::warning::Skipping promotion of ${{ github.sha }}: ${GITHUB_REF_NAME} has moved to ${HEAD_SHA}. Moving the deploy tags to this commit would deploy stale code; push a revert commit to roll back instead."
+            echo "fresh=false" >> $GITHUB_OUTPUT
+          else
+            echo "fresh=true" >> $GITHUB_OUTPUT
+          fi
 
       - name: Promote images to deploy tags
+        if: steps.guard.outputs.fresh == 'true'
+        env:
+          ECR_REPOS: >-
+            ${{ secrets.ECR_APP }}
+            ${{ secrets.ECR_MIGRATIONS }}
+            ${{ secrets.ECR_REALTIME }}
+            ${{ secrets.ECR_PII }}
         run: |
-          ECR_IMAGE="${{ steps.login-ecr.outputs.registry }}/${{ steps.ecr-repo.outputs.name }}"
-          GHCR_IMAGE="${{ matrix.ghcr_image }}"
+          REGISTRY="${{ steps.login-ecr.outputs.registry }}"
 
           if [ "${{ github.ref }}" = "refs/heads/main" ]; then
             ECR_TAG="latest"
@@ -334,29 +330,26 @@ jobs:
             ECR_TAG="staging"
           fi
 
-          echo "🚀 Promoting ${ECR_IMAGE}:${{ github.sha }} to ${ECR_TAG}"
-          docker buildx imagetools create \
-            -t "${ECR_IMAGE}:${ECR_TAG}" \
-            "${ECR_IMAGE}:${{ github.sha }}"
+          # Verify every sha image exists before moving any deploy tag, so a
+          # missing/expired image aborts the whole promotion up front.
+          for repo in $ECR_REPOS; do
+            echo "🔍 Verifying ${repo}:${{ github.sha }}"
+            docker buildx imagetools inspect "${REGISTRY}/${repo}:${{ github.sha }}" > /dev/null
+          done
 
-          if [ "${{ github.ref }}" = "refs/heads/main" ] && [ -n "$GHCR_IMAGE" ]; then
+          for repo in $ECR_REPOS; do
+            echo "🚀 Promoting ${repo}:${{ github.sha }} to ${ECR_TAG}"
             docker buildx imagetools create \
-              -t "${GHCR_IMAGE}:latest-amd64" \
-              "${GHCR_IMAGE}:${{ github.sha }}-amd64"
-
-            if [ "${{ needs.detect-version.outputs.is_release }}" = "true" ]; then
-              VERSION="${{ needs.detect-version.outputs.version }}"
-              echo "📦 Adding version tag: ${VERSION}-amd64"
-              docker buildx imagetools create \
-                -t "${GHCR_IMAGE}:${VERSION}-amd64" \
-                "${GHCR_IMAGE}:${{ github.sha }}-amd64"
-            fi
-          fi
-
-  # Build ARM64 images for GHCR (main branch only, runs in parallel)
+              -t "${REGISTRY}/${repo}:${ECR_TAG}" \
+              "${REGISTRY}/${repo}:${{ github.sha }}"
+          done
+
+  # Build ARM64 images for GHCR (main branch only, runs in parallel with
+  # tests). Pushes only the immutable sha tag — latest-arm64/version-arm64
+  # are applied by create-ghcr-manifests after the gate, so a failing run
+  # never moves a documented tag.
   build-ghcr-arm64:
     name: Build ARM64 (GHCR Only)
-    needs: [detect-version]
     runs-on: blacksmith-8vcpu-ubuntu-2404-arm
     if: github.event_name == 'push' && github.ref == 'refs/heads/main'
     permissions:
@@ -389,21 +382,6 @@ jobs:
       - name: Set up Docker Buildx
         uses: useblacksmith/setup-docker-builder@ab5c1da94f53f5cd75c1038092aa276dddfccbba # v1
 
-      - name: Generate ARM64 tags
-        id: meta
-        run: |
-          IMAGE="${{ matrix.image }}"
-          TAGS="${IMAGE}:latest-arm64,${IMAGE}:${{ github.sha }}-arm64"
-
-          # Add version tag if this is a release commit
-          if [ "${{ needs.detect-version.outputs.is_release }}" = "true" ]; then
-            VERSION="${{ needs.detect-version.outputs.version }}"
-            TAGS="${TAGS},${IMAGE}:${VERSION}-arm64"
-            echo "📦 Adding version tag: ${VERSION}-arm64"
-          fi
-
-          echo "tags=${TAGS}" >> $GITHUB_OUTPUT
-
       - name: Build and push ARM64 to GHCR
         uses: useblacksmith/build-push-action@fb9e3e6a9299c78462bfadd0d93352c316adc9b8 # v2
         with:
@@ -411,11 +389,13 @@ jobs:
           file: ${{ matrix.dockerfile }}
           platforms: linux/arm64
           push: true
-          tags: ${{ steps.meta.outputs.tags }}
+          tags: ${{ matrix.image }}:${{ github.sha }}-arm64
           provenance: false
           sbom: false
 
-  # Create GHCR multi-arch manifests (only for main, after both builds)
+  # Publish all mutable GHCR tags (latest, latest-amd64/arm64, version tags)
+  # and the multi-arch manifests from the immutable sha tags — only on main,
+  # after the deploy gate (promote-images) and the ARM64 build both pass.
   create-ghcr-manifests:
     name: Create GHCR Manifests
     runs-on: blacksmith-2vcpu-ubuntu-2404
@@ -439,30 +419,43 @@ jobs:
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Create and push manifests
+      # Same protection as promote-images: a re-run of an old run must not
+      # move the public latest tags back to a stale commit. Immutable tags
+      # (sha and version) are still published — only latest moves are gated.
+      - name: Guard against stale latest tags
+        id: guard
         run: |
-          IMAGE_BASE="${{ matrix.image }}"
+          HEAD_SHA="$(git ls-remote "https://github.com/${{ github.repository }}.git" "refs/heads/${GITHUB_REF_NAME}" | cut -f1)"
+          if [ "$HEAD_SHA" != "${{ github.sha }}" ]; then
+            echo "::warning::${GITHUB_REF_NAME} has moved to ${HEAD_SHA}; publishing immutable tags for ${{ github.sha }} but skipping the latest tags."
+            echo "fresh=false" >> $GITHUB_OUTPUT
+          else
+            echo "fresh=true" >> $GITHUB_OUTPUT
+          fi
 
-          # Create latest manifest
-          docker manifest create "${IMAGE_BASE}:latest" \
-            "${IMAGE_BASE}:latest-amd64" \
-            "${IMAGE_BASE}:latest-arm64"
-          docker manifest push "${IMAGE_BASE}:latest"
+      - name: Publish tags and manifests
+        run: |
+          IMAGE="${{ matrix.image }}"
+          SHA="${{ github.sha }}"
 
-          # Create SHA manifest
-          docker manifest create "${IMAGE_BASE}:${{ github.sha }}" \
-            "${IMAGE_BASE}:${{ github.sha }}-amd64" \
-            "${IMAGE_BASE}:${{ github.sha }}-arm64"
-          docker manifest push "${IMAGE_BASE}:${{ github.sha }}"
+          # Multi-arch manifest from the immutable per-arch sha tags
+          docker buildx imagetools create -t "${IMAGE}:${SHA}" \
+            "${IMAGE}:${SHA}-amd64" "${IMAGE}:${SHA}-arm64"
 
-          # Create version manifest if this is a release commit
           if [ "${{ needs.detect-version.outputs.is_release }}" = "true" ]; then
             VERSION="${{ needs.detect-version.outputs.version }}"
-            echo "📦 Creating version manifest: ${VERSION}"
-            docker manifest create "${IMAGE_BASE}:${VERSION}" \
-              "${IMAGE_BASE}:${VERSION}-amd64" \
-              "${IMAGE_BASE}:${VERSION}-arm64"
-            docker manifest push "${IMAGE_BASE}:${VERSION}"
+            echo "📦 Publishing version tags: ${VERSION}"
+            docker buildx imagetools create -t "${IMAGE}:${VERSION}-amd64" "${IMAGE}:${SHA}-amd64"
+            docker buildx imagetools create -t "${IMAGE}:${VERSION}-arm64" "${IMAGE}:${SHA}-arm64"
+            docker buildx imagetools create -t "${IMAGE}:${VERSION}" \
+              "${IMAGE}:${SHA}-amd64" "${IMAGE}:${SHA}-arm64"
+          fi
+
+          if [ "${{ steps.guard.outputs.fresh }}" = "true" ]; then
+            docker buildx imagetools create -t "${IMAGE}:latest-amd64" "${IMAGE}:${SHA}-amd64"
+            docker buildx imagetools create -t "${IMAGE}:latest-arm64" "${IMAGE}:${SHA}-arm64"
+            docker buildx imagetools create -t "${IMAGE}:latest" \
+              "${IMAGE}:${SHA}-amd64" "${IMAGE}:${SHA}-arm64"
           fi
 
   # Check if docs changed
diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml
index 239d0586d7f..5c427b3c5e6 100644
--- a/.github/workflows/test-build.yml
+++ b/.github/workflows/test-build.yml
@@ -174,9 +174,13 @@ jobs:
           fail_ci_if_error: false
           verbose: true
 
-  # Next.js production build, in parallel with lint + tests. Uses its own
-  # sticky-disk keys — a sticky disk key can only be mounted by one job at a
-  # time, so sharing keys with test-build would serialize the two jobs.
+  # Next.js production build, in parallel with lint + tests. Sticky disks are
+  # cloned from the last committed snapshot per job and committed last-writer-
+  # wins, so concurrent mounts are safe. The bun/node_modules disks are shared
+  # with test-build (identical content from the same lockfile — LWW loss is
+  # harmless), but the Turbo cache gets its own key: with a shared key, only
+  # the last committer's new entries survive each run, so the test and build
+  # Turbo entries would evict each other nondeterministically.
   build:
     name: Build App
     runs-on: blacksmith-8vcpu-ubuntu-2404
@@ -198,13 +202,13 @@ jobs:
       - name: Mount Bun cache (Sticky Disk)
         uses: useblacksmith/stickydisk@4c034ba57b706cf0e3b4b0ce098c2a3b1071580c # v1
         with:
-          key: ${{ github.repository }}-bun-cache-build
+          key: ${{ github.repository }}-bun-cache
           path: ~/.bun/install/cache
 
       - name: Mount node_modules (Sticky Disk)
         uses: useblacksmith/stickydisk@4c034ba57b706cf0e3b4b0ce098c2a3b1071580c # v1
         with:
-          key: ${{ github.repository }}-node-modules-build
+          key: ${{ github.repository }}-node-modules
           path: ./node_modules
 
       - name: Mount Turbo cache (Sticky Disk)