improvement(ci): decouple image builds from the test gate #22353

Workflow file for this run

	name: CI

	on:
	push:
	branches: [main, staging, dev]
	pull_request:
	branches: [main, staging, dev]

	concurrency:
	group: ci-${{ github.ref }}
	cancel-in-progress: ${{ github.event_name == 'pull_request' }}

	permissions:
	contents: read

	jobs:
	test-build:
	name: Test and Build
	if: github.ref != 'refs/heads/dev' \|\| github.event_name == 'pull_request'
	uses: ./.github/workflows/test-build.yml
	secrets: inherit

	# Detect if this is a version release commit (e.g., "v0.5.24: ...")
	detect-version:
	name: Detect Version
	runs-on: blacksmith-4vcpu-ubuntu-2404
	if: github.event_name == 'push' && (github.ref == 'refs/heads/main' \|\| github.ref == 'refs/heads/staging' \|\| github.ref == 'refs/heads/dev')
	outputs:
	version: ${{ steps.extract.outputs.version }}
	is_release: ${{ steps.extract.outputs.is_release }}
	steps:
	- name: Extract version from commit message
	id: extract
	env:
	COMMIT_MSG: ${{ github.event.head_commit.message }}
	run: \|
	# Only tag versions on main branch
	if [ "$GITHUB_REF" = "refs/heads/main" ] && [[ "$COMMIT_MSG" =~ ^(v[0-9]+\.[0-9]+\.[0-9]+): ]]; then
	VERSION="${BASH_REMATCH[1]}"
	echo "version=${VERSION}" >> $GITHUB_OUTPUT
	echo "is_release=true" >> $GITHUB_OUTPUT
	echo "✅ Detected release commit: ${VERSION}"
	else
	echo "version=" >> $GITHUB_OUTPUT
	echo "is_release=false" >> $GITHUB_OUTPUT
	echo "ℹ️ Not a release commit"
	fi

	# Run database migrations before images are promoted: the ECR latest/staging
	# tag push triggers CodePipeline, so migrating first guarantees the schema is
	# in place before the new app version deploys (replaces the removed ECS
	# migration sidecar)
	migrate:
	name: Migrate DB
	needs: [test-build]
	if: >-
	github.event_name == 'push' &&
	(github.ref == 'refs/heads/main' \|\| github.ref == 'refs/heads/staging')
	uses: ./.github/workflows/migrations.yml
	with:
	environment: ${{ github.ref == 'refs/heads/main' && 'production' \|\| 'staging' }}
	secrets: inherit

	# Same ordering for dev (schema push before the dev image lands in ECR)
	migrate-dev:
	name: Migrate Dev DB
	if: github.event_name == 'push' && github.ref == 'refs/heads/dev'
	uses: ./.github/workflows/migrations.yml
	with:
	environment: dev
	secrets: inherit

	# Dev: build all 3 images for ECR only (no GHCR, no ARM64)
	build-dev:
	name: Build Dev ECR
	needs: [detect-version, migrate-dev]
	if: github.event_name == 'push' && github.ref == 'refs/heads/dev'
	runs-on: blacksmith-8vcpu-ubuntu-2404
	permissions:
	contents: read
	id-token: write
	strategy:
	fail-fast: false
	matrix:
	include:
	- dockerfile: ./docker/app.Dockerfile
	ecr_repo_secret: ECR_APP
	- dockerfile: ./docker/db.Dockerfile
	ecr_repo_secret: ECR_MIGRATIONS
	- dockerfile: ./docker/realtime.Dockerfile
	ecr_repo_secret: ECR_REALTIME
	- dockerfile: ./docker/pii.Dockerfile
	ecr_repo_secret: ECR_PII
	steps:
	- name: Checkout code
	uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6

	- name: Configure AWS credentials
	uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6
	with:
	role-to-assume: ${{ secrets.DEV_AWS_ROLE_TO_ASSUME }}
	aws-region: ${{ secrets.DEV_AWS_REGION }}

	- name: Login to Amazon ECR
	id: login-ecr
	uses: aws-actions/amazon-ecr-login@d539f0932e70871a027e9d5a9d8fc38589180a64 # v2

	- name: Login to Docker Hub
	uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4
	with:
	username: ${{ secrets.DOCKERHUB_USERNAME }}
	password: ${{ secrets.DOCKERHUB_TOKEN }}

	- name: Set up Docker Buildx
	uses: useblacksmith/setup-docker-builder@ab5c1da94f53f5cd75c1038092aa276dddfccbba # v1

	- name: Resolve ECR repo name
	id: ecr-repo
	run: echo "name=$ECR_REPO" >> $GITHUB_OUTPUT
	env:
	ECR_REPO: ${{ matrix.ecr_repo_secret == 'ECR_APP' && secrets.ECR_APP \|\| matrix.ecr_repo_secret == 'ECR_MIGRATIONS' && secrets.ECR_MIGRATIONS \|\| matrix.ecr_repo_secret == 'ECR_REALTIME' && secrets.ECR_REALTIME \|\| matrix.ecr_repo_secret == 'ECR_PII' && secrets.ECR_PII \|\| '' }}

	- name: Build and push
	uses: useblacksmith/build-push-action@fb9e3e6a9299c78462bfadd0d93352c316adc9b8 # v2
	with:
	context: .
	file: ${{ matrix.dockerfile }}
	platforms: linux/amd64
	push: true
	tags: ${{ steps.login-ecr.outputs.registry }}/${{ steps.ecr-repo.outputs.name }}:dev
	provenance: false
	sbom: false

	# Dev: deploy Trigger.dev background tasks to the preview "dev-sim" branch.
	# Gated after migrate-dev for the same reason as build-dev — the new task
	# code runs against the dev DB, so the schema must be pushed first.
	deploy-trigger-dev:
	name: Deploy Trigger.dev (Dev)
	needs: [migrate-dev]
	if: github.event_name == 'push' && github.ref == 'refs/heads/dev'
	runs-on: blacksmith-4vcpu-ubuntu-2404
	steps:
	- name: Checkout code
	uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6

	- name: Setup Bun
	uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
	with:
	bun-version: 1.3.13

	- name: Cache Bun dependencies
	uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
	with:
	path: \|
	~/.bun/install/cache
	node_modules
	**/node_modules
	key: ${{ runner.os }}-bun-${{ hashFiles('**/bun.lock') }}
	restore-keys: \|
	${{ runner.os }}-bun-

	- name: Install dependencies
	run: bun install --frozen-lockfile

	- name: Deploy to Trigger.dev
	working-directory: ./apps/sim
	env:
	TRIGGER_ACCESS_TOKEN: ${{ secrets.DEV_TRIGGER_ACCESS_TOKEN }}
	TRIGGER_PROJECT_ID: ${{ secrets.TRIGGER_PROJECT_ID }}
	run: \|
	if [ -z "$TRIGGER_ACCESS_TOKEN" ] \|\| [ -z "$TRIGGER_PROJECT_ID" ]; then
	echo "ERROR: DEV_TRIGGER_ACCESS_TOKEN and TRIGGER_PROJECT_ID repo secrets must both be set" >&2
	exit 1
	fi
	bunx trigger.dev@4.4.3 deploy --env preview --branch dev-sim

	# Main/staging: build AMD64 images and push sha-tagged images to ECR + GHCR.
	# Runs in parallel with tests — only sha tags are pushed here, and the
	# CodePipeline EventBridge triggers filter on exactly the latest/staging/dev
	# ECR tags, so nothing deploys until promote-images retags after the gate.
	build-amd64:
	name: Build AMD64
	needs: [detect-version]
	if: >-
	github.event_name == 'push' &&
	(github.ref == 'refs/heads/main' \|\| github.ref == 'refs/heads/staging')
	runs-on: blacksmith-8vcpu-ubuntu-2404
	permissions:
	contents: read
	packages: write
	id-token: write
	strategy:
	fail-fast: false
	matrix:
	include:
	- dockerfile: ./docker/app.Dockerfile
	ghcr_image: ghcr.io/simstudioai/simstudio
	ecr_repo_secret: ECR_APP
	- dockerfile: ./docker/db.Dockerfile
	ghcr_image: ghcr.io/simstudioai/migrations
	ecr_repo_secret: ECR_MIGRATIONS
	- dockerfile: ./docker/realtime.Dockerfile
	ghcr_image: ghcr.io/simstudioai/realtime
	ecr_repo_secret: ECR_REALTIME
	- dockerfile: ./docker/pii.Dockerfile
	ghcr_image: ghcr.io/simstudioai/pii
	ecr_repo_secret: ECR_PII
	steps:
	- name: Checkout code
	uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6

	- name: Configure AWS credentials
	uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6
	with:
	role-to-assume: ${{ github.ref == 'refs/heads/main' && secrets.AWS_ROLE_TO_ASSUME \|\| secrets.STAGING_AWS_ROLE_TO_ASSUME }}
	aws-region: ${{ github.ref == 'refs/heads/main' && secrets.AWS_REGION \|\| secrets.STAGING_AWS_REGION }}

	- name: Login to Amazon ECR
	id: login-ecr
	uses: aws-actions/amazon-ecr-login@d539f0932e70871a027e9d5a9d8fc38589180a64 # v2

	- name: Login to Docker Hub
	uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4
	with:
	username: ${{ secrets.DOCKERHUB_USERNAME }}
	password: ${{ secrets.DOCKERHUB_TOKEN }}

	- name: Login to GHCR
	if: github.ref == 'refs/heads/main'
	uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4
	with:
	registry: ghcr.io
	username: ${{ github.repository_owner }}
	password: ${{ secrets.GITHUB_TOKEN }}

	- name: Set up Docker Buildx
	uses: useblacksmith/setup-docker-builder@ab5c1da94f53f5cd75c1038092aa276dddfccbba # v1

	- name: Resolve ECR repo name
	id: ecr-repo
	run: echo "name=$ECR_REPO" >> $GITHUB_OUTPUT
	env:
	ECR_REPO: ${{ matrix.ecr_repo_secret == 'ECR_APP' && secrets.ECR_APP \|\| matrix.ecr_repo_secret == 'ECR_MIGRATIONS' && secrets.ECR_MIGRATIONS \|\| matrix.ecr_repo_secret == 'ECR_REALTIME' && secrets.ECR_REALTIME \|\| matrix.ecr_repo_secret == 'ECR_PII' && secrets.ECR_PII \|\| '' }}

	# Only sha tags here — the deploy tags (latest/staging) and the GHCR
	# latest-amd64/version tags are applied by promote-images after tests
	# and migrations pass.
	- name: Generate tags
	id: meta
	run: \|
	ECR_REGISTRY="${{ steps.login-ecr.outputs.registry }}"
	ECR_REPO="${{ steps.ecr-repo.outputs.name }}"
	GHCR_IMAGE="${{ matrix.ghcr_image }}"

	TAGS="${ECR_REGISTRY}/${ECR_REPO}:${{ github.sha }}"

	if [ "${{ github.ref }}" = "refs/heads/main" ] && [ -n "$GHCR_IMAGE" ]; then
	TAGS="${TAGS},${GHCR_IMAGE}:${{ github.sha }}-amd64"
	fi

	echo "tags=${TAGS}" >> $GITHUB_OUTPUT

	- name: Build and push images
	uses: useblacksmith/build-push-action@fb9e3e6a9299c78462bfadd0d93352c316adc9b8 # v2
	with:
	context: .
	file: ${{ matrix.dockerfile }}
	platforms: linux/amd64
	push: true
	tags: ${{ steps.meta.outputs.tags }}
	provenance: false
	sbom: false

	# Promote the sha-tagged images to the deploy tags once tests and migrations
	# pass. Pushing the ECR latest/staging tag is what triggers CodePipeline, so
	# this seconds-long manifest retag is the deploy gate — the image builds
	# themselves run in parallel with the tests.
	promote-images:
	name: Promote Images
	needs: [detect-version, migrate, build-amd64]
	if: >-
	github.event_name == 'push' &&
	(github.ref == 'refs/heads/main' \|\| github.ref == 'refs/heads/staging')
	runs-on: blacksmith-2vcpu-ubuntu-2404
	permissions:
	contents: read
	packages: write
	id-token: write
	strategy:
	fail-fast: false
	matrix:
	include:
	- ghcr_image: ghcr.io/simstudioai/simstudio
	ecr_repo_secret: ECR_APP
	- ghcr_image: ghcr.io/simstudioai/migrations
	ecr_repo_secret: ECR_MIGRATIONS
	- ghcr_image: ghcr.io/simstudioai/realtime
	ecr_repo_secret: ECR_REALTIME
	- ghcr_image: ghcr.io/simstudioai/pii
	ecr_repo_secret: ECR_PII
	steps:
	- name: Configure AWS credentials
	uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6
	with:
	role-to-assume: ${{ github.ref == 'refs/heads/main' && secrets.AWS_ROLE_TO_ASSUME \|\| secrets.STAGING_AWS_ROLE_TO_ASSUME }}
	aws-region: ${{ github.ref == 'refs/heads/main' && secrets.AWS_REGION \|\| secrets.STAGING_AWS_REGION }}

	- name: Login to Amazon ECR
	id: login-ecr
	uses: aws-actions/amazon-ecr-login@d539f0932e70871a027e9d5a9d8fc38589180a64 # v2

	- name: Login to GHCR
	if: github.ref == 'refs/heads/main'
	uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4
	with:
	registry: ghcr.io
	username: ${{ github.repository_owner }}
	password: ${{ secrets.GITHUB_TOKEN }}

	- name: Resolve ECR repo name
	id: ecr-repo
	run: echo "name=$ECR_REPO" >> $GITHUB_OUTPUT
	env:
	ECR_REPO: ${{ matrix.ecr_repo_secret == 'ECR_APP' && secrets.ECR_APP \|\| matrix.ecr_repo_secret == 'ECR_MIGRATIONS' && secrets.ECR_MIGRATIONS \|\| matrix.ecr_repo_secret == 'ECR_REALTIME' && secrets.ECR_REALTIME \|\| matrix.ecr_repo_secret == 'ECR_PII' && secrets.ECR_PII \|\| '' }}

	- name: Promote images to deploy tags
	run: \|
	ECR_IMAGE="${{ steps.login-ecr.outputs.registry }}/${{ steps.ecr-repo.outputs.name }}"
	GHCR_IMAGE="${{ matrix.ghcr_image }}"

	if [ "${{ github.ref }}" = "refs/heads/main" ]; then
	ECR_TAG="latest"
	else
	ECR_TAG="staging"
	fi

	echo "🚀 Promoting ${ECR_IMAGE}:${{ github.sha }} to ${ECR_TAG}"
	docker buildx imagetools create \
	-t "${ECR_IMAGE}:${ECR_TAG}" \
	"${ECR_IMAGE}:${{ github.sha }}"

	if [ "${{ github.ref }}" = "refs/heads/main" ] && [ -n "$GHCR_IMAGE" ]; then
	docker buildx imagetools create \
	-t "${GHCR_IMAGE}:latest-amd64" \
	"${GHCR_IMAGE}:${{ github.sha }}-amd64"

	if [ "${{ needs.detect-version.outputs.is_release }}" = "true" ]; then
	VERSION="${{ needs.detect-version.outputs.version }}"
	echo "📦 Adding version tag: ${VERSION}-amd64"
	docker buildx imagetools create \
	-t "${GHCR_IMAGE}:${VERSION}-amd64" \
	"${GHCR_IMAGE}:${{ github.sha }}-amd64"
	fi
	fi

	# Build ARM64 images for GHCR (main branch only, runs in parallel)
	build-ghcr-arm64:
	name: Build ARM64 (GHCR Only)
	needs: [detect-version]
	runs-on: blacksmith-8vcpu-ubuntu-2404-arm
	if: github.event_name == 'push' && github.ref == 'refs/heads/main'
	permissions:
	contents: read
	packages: write
	strategy:
	fail-fast: false
	matrix:
	include:
	- dockerfile: ./docker/app.Dockerfile
	image: ghcr.io/simstudioai/simstudio
	- dockerfile: ./docker/db.Dockerfile
	image: ghcr.io/simstudioai/migrations
	- dockerfile: ./docker/realtime.Dockerfile
	image: ghcr.io/simstudioai/realtime
	- dockerfile: ./docker/pii.Dockerfile
	image: ghcr.io/simstudioai/pii

	steps:
	- name: Checkout code
	uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6

	- name: Login to GHCR
	uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4
	with:
	registry: ghcr.io
	username: ${{ github.repository_owner }}
	password: ${{ secrets.GITHUB_TOKEN }}

	- name: Set up Docker Buildx
	uses: useblacksmith/setup-docker-builder@ab5c1da94f53f5cd75c1038092aa276dddfccbba # v1

	- name: Generate ARM64 tags
	id: meta
	run: \|
	IMAGE="${{ matrix.image }}"
	TAGS="${IMAGE}:latest-arm64,${IMAGE}:${{ github.sha }}-arm64"

	# Add version tag if this is a release commit
	if [ "${{ needs.detect-version.outputs.is_release }}" = "true" ]; then
	VERSION="${{ needs.detect-version.outputs.version }}"
	TAGS="${TAGS},${IMAGE}:${VERSION}-arm64"
	echo "📦 Adding version tag: ${VERSION}-arm64"
	fi

	echo "tags=${TAGS}" >> $GITHUB_OUTPUT

	- name: Build and push ARM64 to GHCR
	uses: useblacksmith/build-push-action@fb9e3e6a9299c78462bfadd0d93352c316adc9b8 # v2
	with:
	context: .
	file: ${{ matrix.dockerfile }}
	platforms: linux/arm64
	push: true
	tags: ${{ steps.meta.outputs.tags }}
	provenance: false
	sbom: false

	# Create GHCR multi-arch manifests (only for main, after both builds)
	create-ghcr-manifests:
	name: Create GHCR Manifests
	runs-on: blacksmith-2vcpu-ubuntu-2404
	needs: [promote-images, build-ghcr-arm64, detect-version]
	if: github.event_name == 'push' && github.ref == 'refs/heads/main'
	permissions:
	packages: write
	strategy:
	matrix:
	include:
	- image: ghcr.io/simstudioai/simstudio
	- image: ghcr.io/simstudioai/migrations
	- image: ghcr.io/simstudioai/realtime
	- image: ghcr.io/simstudioai/pii

	steps:
	- name: Login to GHCR
	uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4
	with:
	registry: ghcr.io
	username: ${{ github.repository_owner }}
	password: ${{ secrets.GITHUB_TOKEN }}

	- name: Create and push manifests
	run: \|
	IMAGE_BASE="${{ matrix.image }}"

	# Create latest manifest
	docker manifest create "${IMAGE_BASE}:latest" \
	"${IMAGE_BASE}:latest-amd64" \
	"${IMAGE_BASE}:latest-arm64"
	docker manifest push "${IMAGE_BASE}:latest"

	# Create SHA manifest
	docker manifest create "${IMAGE_BASE}:${{ github.sha }}" \
	"${IMAGE_BASE}:${{ github.sha }}-amd64" \
	"${IMAGE_BASE}:${{ github.sha }}-arm64"
	docker manifest push "${IMAGE_BASE}:${{ github.sha }}"

	# Create version manifest if this is a release commit
	if [ "${{ needs.detect-version.outputs.is_release }}" = "true" ]; then
	VERSION="${{ needs.detect-version.outputs.version }}"
	echo "📦 Creating version manifest: ${VERSION}"
	docker manifest create "${IMAGE_BASE}:${VERSION}" \
	"${IMAGE_BASE}:${VERSION}-amd64" \
	"${IMAGE_BASE}:${VERSION}-arm64"
	docker manifest push "${IMAGE_BASE}:${VERSION}"
	fi

	# Check if docs changed
	check-docs-changes:
	name: Check Docs Changes
	runs-on: blacksmith-4vcpu-ubuntu-2404
	if: github.event_name == 'push' && github.ref == 'refs/heads/main'
	outputs:
	docs_changed: ${{ steps.filter.outputs.docs }}
	steps:
	- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
	with:
	fetch-depth: 2 # Need at least 2 commits to detect changes
	- uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4
	id: filter
	with:
	filters: \|
	docs:
	- 'apps/docs/content/docs/en/**'
	- 'apps/sim/scripts/process-docs.ts'
	- 'apps/sim/lib/chunkers/**'

	# Process docs embeddings (only when docs change, after images are promoted)
	process-docs:
	name: Process Docs
	needs: [promote-images, check-docs-changes]
	if: needs.check-docs-changes.outputs.docs_changed == 'true'
	uses: ./.github/workflows/docs-embeddings.yml
	secrets: inherit

	# Create GitHub Release (only for version commits on main, after all builds complete)
	create-release:
	name: Create GitHub Release
	runs-on: blacksmith-4vcpu-ubuntu-2404
	needs: [create-ghcr-manifests, detect-version]
	if: needs.detect-version.outputs.is_release == 'true'
	permissions:
	contents: write
	steps:
	- name: Checkout code
	uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
	with:
	fetch-depth: 0

	- name: Setup Bun
	uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
	with:
	bun-version: 1.3.13

	- name: Install dependencies
	run: bun install --frozen-lockfile

	- name: Create release
	env:
	GH_PAT: ${{ secrets.GITHUB_TOKEN }}
	run: bun run scripts/create-single-release.ts ${{ needs.detect-version.outputs.version }}

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

improvement(ci): decouple image builds from the test gate #22353

Workflow file

improvement(ci): decouple image builds from the test gate #22353

Uh oh!

Workflow file for this run