improvement(ci): decouple image builds from the test gate (#5701)

waleedlatif1 · web-flow · commit 4bb560073f75 · 2026-07-15T22:43:43.000-07:00
* improvement(ci): decouple image builds from the test gate

- build-amd64 now starts immediately and pushes only sha tags (ECR :sha,
  GHCR :sha-amd64). The EventBridge deploy triggers filter on exactly the
  latest/staging/dev ECR tags, so nothing deploys from these pushes.
- New promote-images job retags sha -&gt; latest/staging (and GHCR
  latest-amd64/version-amd64) via buildx imagetools once tests and
  migrations pass — a seconds-long manifest copy instead of rebuilding
  after the gate. Cuts push-to-deploy from ~13.5 to ~7 minutes.
- Split the Next.js production build out of test-build into a parallel
  Build App job with its own sticky-disk keys, cutting PR feedback from
  ~5.5 to ~3.5 minutes.
- create-ghcr-manifests and process-docs now gate on promote-images.

* improvement(ci): harden promotion — atomic retag, stale-run guards, gate all mutable GHCR tags

Review follow-ups:
- promote-images is a single job (not a matrix): verifies all four :sha
  manifests exist before moving any deploy tag, so a missing image can't
  cause a partial mixed-version deploy
- stale-run guard on promote-images and create-ghcr-manifests: re-running
  an old run no longer retags latest/staging back to stale code (a
  one-click prod rollback); superseded runs skip mutable tags with a
  warning while immutable sha/version tags still publish
- ARM64 build now pushes only the immutable :sha-arm64 tag; latest-arm64
  and version tags moved behind the gate into create-ghcr-manifests
  (closes the pre-existing hole where a failing run moved latest-arm64)
- dropped dead detect-version needs from both build jobs
- sticky-disk comment corrected (clone + last-writer-wins, not exclusive
  mounts); build job shares warm bun/node_modules disks, keeps its own
  turbo cache key so test/build entries don't evict each other
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -46,9 +46,10 @@ jobs:
             echo "ℹ️ Not a release commit"
           fi
 
-  # Run database migrations before images are pushed: the ECR push triggers
-  # CodePipeline, so migrating first guarantees the schema is in place before
-  # the new app version deploys (replaces the removed ECS migration sidecar)
+  # Run database migrations before images are promoted: the ECR latest/staging
+  # tag push triggers CodePipeline, so migrating first guarantees the schema is
+  # in place before the new app version deploys (replaces the removed ECS
+  # migration sidecar)
   migrate:
     name: Migrate DB
     needs: [test-build]
@@ -173,10 +174,13 @@ jobs:
           fi
           bunx trigger.dev@4.4.3 deploy --env preview --branch dev-sim
 
-  # Main/staging: build AMD64 images and push to ECR + GHCR
+  # Main/staging: build AMD64 images and push sha-tagged images to ECR + GHCR.
+  # Runs in parallel with tests — only immutable sha tags are pushed here, and
+  # the CodePipeline EventBridge triggers filter on exactly the
+  # latest/staging/dev ECR tags, so nothing deploys and no mutable tag moves
+  # until promote-images / create-ghcr-manifests retag after the gate.
   build-amd64:
     name: Build AMD64
-    needs: [test-build, detect-version, migrate]
     if: >-
       github.event_name == 'push' &&
       (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging')
@@ -238,33 +242,20 @@ jobs:
         env:
           ECR_REPO: ${{ matrix.ecr_repo_secret == 'ECR_APP' && secrets.ECR_APP || matrix.ecr_repo_secret == 'ECR_MIGRATIONS' && secrets.ECR_MIGRATIONS || matrix.ecr_repo_secret == 'ECR_REALTIME' && secrets.ECR_REALTIME || matrix.ecr_repo_secret == 'ECR_PII' && secrets.ECR_PII || '' }}
 
+      # Only sha tags here — the ECR deploy tags (latest/staging) are applied
+      # by promote-images and the GHCR latest-amd64/version tags by
+      # create-ghcr-manifests, both after tests and migrations pass.
       - name: Generate tags
         id: meta
         run: |
           ECR_REGISTRY="${{ steps.login-ecr.outputs.registry }}"
           ECR_REPO="${{ steps.ecr-repo.outputs.name }}"
           GHCR_IMAGE="${{ matrix.ghcr_image }}"
 
-          if [ "${{ github.ref }}" = "refs/heads/main" ]; then
-            ECR_TAG="latest"
-          else
-            ECR_TAG="staging"
-          fi
-          ECR_IMAGE="${ECR_REGISTRY}/${ECR_REPO}:${ECR_TAG}"
-
-          TAGS="${ECR_IMAGE}"
+          TAGS="${ECR_REGISTRY}/${ECR_REPO}:${{ github.sha }}"
 
           if [ "${{ github.ref }}" = "refs/heads/main" ] && [ -n "$GHCR_IMAGE" ]; then
-            GHCR_AMD64="${GHCR_IMAGE}:latest-amd64"
-            GHCR_SHA="${GHCR_IMAGE}:${{ github.sha }}-amd64"
-            TAGS="${TAGS},$GHCR_AMD64,$GHCR_SHA"
-
-            if [ "${{ needs.detect-version.outputs.is_release }}" = "true" ]; then
-              VERSION="${{ needs.detect-version.outputs.version }}"
-              GHCR_VERSION="${GHCR_IMAGE}:${VERSION}-amd64"
-              TAGS="${TAGS},$GHCR_VERSION"
-              echo "📦 Adding version tag: ${VERSION}-amd64"
-            fi
+            TAGS="${TAGS},${GHCR_IMAGE}:${{ github.sha }}-amd64"
           fi
 
           echo "tags=${TAGS}" >> $GITHUB_OUTPUT
@@ -280,10 +271,85 @@ jobs:
           provenance: false
           sbom: false
 
-  # Build ARM64 images for GHCR (main branch only, runs in parallel)
+  # Promote the sha-tagged ECR images to the deploy tags once tests and
+  # migrations pass. Pushing the ECR latest/staging tag is what triggers
+  # CodePipeline, so this seconds-long manifest retag is the deploy gate —
+  # the image builds themselves run in parallel with the tests. A single job
+  # (not a matrix) so all four sha manifests are verified before any tag
+  # moves; a missing image can't produce a partial mixed-version deploy.
+  promote-images:
+    name: Promote Images
+    needs: [migrate, build-amd64]
+    if: >-
+      github.event_name == 'push' &&
+      (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging')
+    runs-on: blacksmith-2vcpu-ubuntu-2404
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6
+        with:
+          role-to-assume: ${{ github.ref == 'refs/heads/main' && secrets.AWS_ROLE_TO_ASSUME || secrets.STAGING_AWS_ROLE_TO_ASSUME }}
+          aws-region: ${{ github.ref == 'refs/heads/main' && secrets.AWS_REGION || secrets.STAGING_AWS_REGION }}
+
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@d539f0932e70871a027e9d5a9d8fc38589180a64 # v2
+
+      # Re-running this job from an old run would retag the deploy tags back
+      # to that run's commit and immediately trigger a production deploy of
+      # stale code. Only promote while this commit is still the branch head;
+      # when superseded, skip cleanly — the newer run's promotion covers it.
+      - name: Guard against stale promotion
+        id: guard
+        run: |
+          HEAD_SHA="$(git ls-remote "https://github.com/${{ github.repository }}.git" "refs/heads/${GITHUB_REF_NAME}" | cut -f1)"
+          if [ "$HEAD_SHA" != "${{ github.sha }}" ]; then
+            echo "::warning::Skipping promotion of ${{ github.sha }}: ${GITHUB_REF_NAME} has moved to ${HEAD_SHA}. Moving the deploy tags to this commit would deploy stale code; push a revert commit to roll back instead."
+            echo "fresh=false" >> $GITHUB_OUTPUT
+          else
+            echo "fresh=true" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Promote images to deploy tags
+        if: steps.guard.outputs.fresh == 'true'
+        env:
+          ECR_REPOS: >-
+            ${{ secrets.ECR_APP }}
+            ${{ secrets.ECR_MIGRATIONS }}
+            ${{ secrets.ECR_REALTIME }}
+            ${{ secrets.ECR_PII }}
+        run: |
+          REGISTRY="${{ steps.login-ecr.outputs.registry }}"
+
+          if [ "${{ github.ref }}" = "refs/heads/main" ]; then
+            ECR_TAG="latest"
+          else
+            ECR_TAG="staging"
+          fi
+
+          # Verify every sha image exists before moving any deploy tag, so a
+          # missing/expired image aborts the whole promotion up front.
+          for repo in $ECR_REPOS; do
+            echo "🔍 Verifying ${repo}:${{ github.sha }}"
+            docker buildx imagetools inspect "${REGISTRY}/${repo}:${{ github.sha }}" > /dev/null
+          done
+
+          for repo in $ECR_REPOS; do
+            echo "🚀 Promoting ${repo}:${{ github.sha }} to ${ECR_TAG}"
+            docker buildx imagetools create \
+              -t "${REGISTRY}/${repo}:${ECR_TAG}" \
+              "${REGISTRY}/${repo}:${{ github.sha }}"
+          done
+
+  # Build ARM64 images for GHCR (main branch only, runs in parallel with
+  # tests). Pushes only the immutable sha tag — latest-arm64/version-arm64
+  # are applied by create-ghcr-manifests after the gate, so a failing run
+  # never moves a documented tag.
   build-ghcr-arm64:
     name: Build ARM64 (GHCR Only)
-    needs: [detect-version]
     runs-on: blacksmith-8vcpu-ubuntu-2404-arm
     if: github.event_name == 'push' && github.ref == 'refs/heads/main'
     permissions:
@@ -316,37 +382,24 @@ jobs:
       - name: Set up Docker Buildx
         uses: useblacksmith/setup-docker-builder@ab5c1da94f53f5cd75c1038092aa276dddfccbba # v1
 
-      - name: Generate ARM64 tags
-        id: meta
-        run: |
-          IMAGE="${{ matrix.image }}"
-          TAGS="${IMAGE}:latest-arm64,${IMAGE}:${{ github.sha }}-arm64"
-
-          # Add version tag if this is a release commit
-          if [ "${{ needs.detect-version.outputs.is_release }}" = "true" ]; then
-            VERSION="${{ needs.detect-version.outputs.version }}"
-            TAGS="${TAGS},${IMAGE}:${VERSION}-arm64"
-            echo "📦 Adding version tag: ${VERSION}-arm64"
-          fi
-
-          echo "tags=${TAGS}" >> $GITHUB_OUTPUT
-
       - name: Build and push ARM64 to GHCR
         uses: useblacksmith/build-push-action@fb9e3e6a9299c78462bfadd0d93352c316adc9b8 # v2
         with:
           context: .
           file: ${{ matrix.dockerfile }}
           platforms: linux/arm64
           push: true
-          tags: ${{ steps.meta.outputs.tags }}
+          tags: ${{ matrix.image }}:${{ github.sha }}-arm64
           provenance: false
           sbom: false
 
-  # Create GHCR multi-arch manifests (only for main, after both builds)
+  # Publish all mutable GHCR tags (latest, latest-amd64/arm64, version tags)
+  # and the multi-arch manifests from the immutable sha tags — only on main,
+  # after the deploy gate (promote-images) and the ARM64 build both pass.
   create-ghcr-manifests:
     name: Create GHCR Manifests
     runs-on: blacksmith-2vcpu-ubuntu-2404
-    needs: [build-amd64, build-ghcr-arm64, detect-version]
+    needs: [promote-images, build-ghcr-arm64, detect-version]
     if: github.event_name == 'push' && github.ref == 'refs/heads/main'
     permissions:
       packages: write
@@ -366,30 +419,43 @@ jobs:
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Create and push manifests
+      # Same protection as promote-images: a re-run of an old run must not
+      # move the public latest tags back to a stale commit. Immutable tags
+      # (sha and version) are still published — only latest moves are gated.
+      - name: Guard against stale latest tags
+        id: guard
         run: |
-          IMAGE_BASE="${{ matrix.image }}"
+          HEAD_SHA="$(git ls-remote "https://github.com/${{ github.repository }}.git" "refs/heads/${GITHUB_REF_NAME}" | cut -f1)"
+          if [ "$HEAD_SHA" != "${{ github.sha }}" ]; then
+            echo "::warning::${GITHUB_REF_NAME} has moved to ${HEAD_SHA}; publishing immutable tags for ${{ github.sha }} but skipping the latest tags."
+            echo "fresh=false" >> $GITHUB_OUTPUT
+          else
+            echo "fresh=true" >> $GITHUB_OUTPUT
+          fi
 
-          # Create latest manifest
-          docker manifest create "${IMAGE_BASE}:latest" \
-            "${IMAGE_BASE}:latest-amd64" \
-            "${IMAGE_BASE}:latest-arm64"
-          docker manifest push "${IMAGE_BASE}:latest"
+      - name: Publish tags and manifests
+        run: |
+          IMAGE="${{ matrix.image }}"
+          SHA="${{ github.sha }}"
 
-          # Create SHA manifest
-          docker manifest create "${IMAGE_BASE}:${{ github.sha }}" \
-            "${IMAGE_BASE}:${{ github.sha }}-amd64" \
-            "${IMAGE_BASE}:${{ github.sha }}-arm64"
-          docker manifest push "${IMAGE_BASE}:${{ github.sha }}"
+          # Multi-arch manifest from the immutable per-arch sha tags
+          docker buildx imagetools create -t "${IMAGE}:${SHA}" \
+            "${IMAGE}:${SHA}-amd64" "${IMAGE}:${SHA}-arm64"
 
-          # Create version manifest if this is a release commit
           if [ "${{ needs.detect-version.outputs.is_release }}" = "true" ]; then
             VERSION="${{ needs.detect-version.outputs.version }}"
-            echo "📦 Creating version manifest: ${VERSION}"
-            docker manifest create "${IMAGE_BASE}:${VERSION}" \
-              "${IMAGE_BASE}:${VERSION}-amd64" \
-              "${IMAGE_BASE}:${VERSION}-arm64"
-            docker manifest push "${IMAGE_BASE}:${VERSION}"
+            echo "📦 Publishing version tags: ${VERSION}"
+            docker buildx imagetools create -t "${IMAGE}:${VERSION}-amd64" "${IMAGE}:${SHA}-amd64"
+            docker buildx imagetools create -t "${IMAGE}:${VERSION}-arm64" "${IMAGE}:${SHA}-arm64"
+            docker buildx imagetools create -t "${IMAGE}:${VERSION}" \
+              "${IMAGE}:${SHA}-amd64" "${IMAGE}:${SHA}-arm64"
+          fi
+
+          if [ "${{ steps.guard.outputs.fresh }}" = "true" ]; then
+            docker buildx imagetools create -t "${IMAGE}:latest-amd64" "${IMAGE}:${SHA}-amd64"
+            docker buildx imagetools create -t "${IMAGE}:latest-arm64" "${IMAGE}:${SHA}-arm64"
+            docker buildx imagetools create -t "${IMAGE}:latest" \
+              "${IMAGE}:${SHA}-amd64" "${IMAGE}:${SHA}-arm64"
           fi
 
   # Check if docs changed
@@ -412,10 +478,10 @@ jobs:
               - 'apps/sim/scripts/process-docs.ts'
               - 'apps/sim/lib/chunkers/**'
 
-  # Process docs embeddings (only when docs change, after ECR images are pushed)
+  # Process docs embeddings (only when docs change, after images are promoted)
   process-docs:
     name: Process Docs
-    needs: [build-amd64, check-docs-changes]
+    needs: [promote-images, check-docs-changes]
     if: needs.check-docs-changes.outputs.docs_changed == 'true'
     uses: ./.github/workflows/docs-embeddings.yml
     secrets: inherit
diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml
@@ -9,7 +9,7 @@ permissions:
 
 jobs:
   test-build:
-    name: Test and Build
+    name: Lint and Test
     runs-on: blacksmith-8vcpu-ubuntu-2404
 
     steps:
@@ -44,14 +44,6 @@ jobs:
           key: ${{ github.repository }}-turbo-cache
           path: ./.turbo
 
-      - name: Restore Next.js build cache
-        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
-        with:
-          path: ./apps/sim/.next/cache
-          key: ${{ runner.os }}-nextjs-${{ hashFiles('bun.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-nextjs-
-
       - name: Install dependencies
         run: bun install --frozen-lockfile
 
@@ -175,6 +167,67 @@ jobs:
           fi
           echo "✅ Schema and migrations are in sync"
 
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5
+        with:
+          directory: ./apps/sim/coverage
+          fail_ci_if_error: false
+          verbose: true
+
+  # Next.js production build, in parallel with lint + tests. Sticky disks are
+  # cloned from the last committed snapshot per job and committed last-writer-
+  # wins, so concurrent mounts are safe. The bun/node_modules disks are shared
+  # with test-build (identical content from the same lockfile — LWW loss is
+  # harmless), but the Turbo cache gets its own key: with a shared key, only
+  # the last committer's new entries survive each run, so the test and build
+  # Turbo entries would evict each other nondeterministically.
+  build:
+    name: Build App
+    runs-on: blacksmith-8vcpu-ubuntu-2404
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
+
+      - name: Setup Bun
+        uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
+        with:
+          bun-version: 1.3.13
+
+      - name: Setup Node
+        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
+        with:
+          node-version: latest
+
+      - name: Mount Bun cache (Sticky Disk)
+        uses: useblacksmith/stickydisk@4c034ba57b706cf0e3b4b0ce098c2a3b1071580c # v1
+        with:
+          key: ${{ github.repository }}-bun-cache
+          path: ~/.bun/install/cache
+
+      - name: Mount node_modules (Sticky Disk)
+        uses: useblacksmith/stickydisk@4c034ba57b706cf0e3b4b0ce098c2a3b1071580c # v1
+        with:
+          key: ${{ github.repository }}-node-modules
+          path: ./node_modules
+
+      - name: Mount Turbo cache (Sticky Disk)
+        uses: useblacksmith/stickydisk@4c034ba57b706cf0e3b4b0ce098c2a3b1071580c # v1
+        with:
+          key: ${{ github.repository }}-turbo-cache-build
+          path: ./.turbo
+
+      - name: Restore Next.js build cache
+        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
+        with:
+          path: ./apps/sim/.next/cache
+          key: ${{ runner.os }}-nextjs-${{ hashFiles('bun.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-nextjs-
+
+      - name: Install dependencies
+        run: bun install --frozen-lockfile
+
       - name: Build application
         env:
           NODE_OPTIONS: '--no-warnings --max-old-space-size=8192'
@@ -186,11 +239,4 @@ jobs:
           AWS_REGION: 'us-west-2'
           ENCRYPTION_KEY: '7cf672e460e430c1fba707575c2b0e2ad5a99dddf9b7b7e3b5646e630861db1c' # dummy key for CI only
           TURBO_CACHE_DIR: .turbo
-        run: bunx turbo run build --filter=sim
-
-      - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5
-        with:
-          directory: ./apps/sim/coverage
-          fail_ci_if_error: false
-          verbose: true
+        run: bunx turbo run build --filter=sim
