diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 25d8348d8c4..88487ace2df 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,9 +46,10 @@ jobs: echo "ℹī¸ Not a release commit" fi - # Run database migrations before images are pushed: the ECR push triggers - # CodePipeline, so migrating first guarantees the schema is in place before - # the new app version deploys (replaces the removed ECS migration sidecar) + # Run database migrations before images are promoted: the ECR latest/staging + # tag push triggers CodePipeline, so migrating first guarantees the schema is + # in place before the new app version deploys (replaces the removed ECS + # migration sidecar) migrate: name: Migrate DB needs: [test-build] @@ -173,10 +174,13 @@ jobs: fi bunx trigger.dev@4.4.3 deploy --env preview --branch dev-sim - # Main/staging: build AMD64 images and push to ECR + GHCR + # Main/staging: build AMD64 images and push sha-tagged images to ECR + GHCR. + # Runs in parallel with tests — only immutable sha tags are pushed here, and + # the CodePipeline EventBridge triggers filter on exactly the + # latest/staging/dev ECR tags, so nothing deploys and no mutable tag moves + # until promote-images / create-ghcr-manifests retag after the gate. build-amd64: name: Build AMD64 - needs: [test-build, detect-version, migrate] if: >- github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging') @@ -238,6 +242,9 @@ jobs: env: ECR_REPO: ${{ matrix.ecr_repo_secret == 'ECR_APP' && secrets.ECR_APP || matrix.ecr_repo_secret == 'ECR_MIGRATIONS' && secrets.ECR_MIGRATIONS || matrix.ecr_repo_secret == 'ECR_REALTIME' && secrets.ECR_REALTIME || matrix.ecr_repo_secret == 'ECR_PII' && secrets.ECR_PII || '' }} + # Only sha tags here — the ECR deploy tags (latest/staging) are applied + # by promote-images and the GHCR latest-amd64/version tags by + # create-ghcr-manifests, both after tests and migrations pass. - name: Generate tags id: meta run: | @@ -245,26 +252,10 @@ jobs: ECR_REPO="${{ steps.ecr-repo.outputs.name }}" GHCR_IMAGE="${{ matrix.ghcr_image }}" - if [ "${{ github.ref }}" = "refs/heads/main" ]; then - ECR_TAG="latest" - else - ECR_TAG="staging" - fi - ECR_IMAGE="${ECR_REGISTRY}/${ECR_REPO}:${ECR_TAG}" - - TAGS="${ECR_IMAGE}" + TAGS="${ECR_REGISTRY}/${ECR_REPO}:${{ github.sha }}" if [ "${{ github.ref }}" = "refs/heads/main" ] && [ -n "$GHCR_IMAGE" ]; then - GHCR_AMD64="${GHCR_IMAGE}:latest-amd64" - GHCR_SHA="${GHCR_IMAGE}:${{ github.sha }}-amd64" - TAGS="${TAGS},$GHCR_AMD64,$GHCR_SHA" - - if [ "${{ needs.detect-version.outputs.is_release }}" = "true" ]; then - VERSION="${{ needs.detect-version.outputs.version }}" - GHCR_VERSION="${GHCR_IMAGE}:${VERSION}-amd64" - TAGS="${TAGS},$GHCR_VERSION" - echo "đŸ“Ļ Adding version tag: ${VERSION}-amd64" - fi + TAGS="${TAGS},${GHCR_IMAGE}:${{ github.sha }}-amd64" fi echo "tags=${TAGS}" >> $GITHUB_OUTPUT @@ -280,10 +271,85 @@ jobs: provenance: false sbom: false - # Build ARM64 images for GHCR (main branch only, runs in parallel) + # Promote the sha-tagged ECR images to the deploy tags once tests and + # migrations pass. Pushing the ECR latest/staging tag is what triggers + # CodePipeline, so this seconds-long manifest retag is the deploy gate — + # the image builds themselves run in parallel with the tests. A single job + # (not a matrix) so all four sha manifests are verified before any tag + # moves; a missing image can't produce a partial mixed-version deploy. + promote-images: + name: Promote Images + needs: [migrate, build-amd64] + if: >- + github.event_name == 'push' && + (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging') + runs-on: blacksmith-2vcpu-ubuntu-2404 + permissions: + contents: read + id-token: write + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6 + with: + role-to-assume: ${{ github.ref == 'refs/heads/main' && secrets.AWS_ROLE_TO_ASSUME || secrets.STAGING_AWS_ROLE_TO_ASSUME }} + aws-region: ${{ github.ref == 'refs/heads/main' && secrets.AWS_REGION || secrets.STAGING_AWS_REGION }} + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@d539f0932e70871a027e9d5a9d8fc38589180a64 # v2 + + # Re-running this job from an old run would retag the deploy tags back + # to that run's commit and immediately trigger a production deploy of + # stale code. Only promote while this commit is still the branch head; + # when superseded, skip cleanly — the newer run's promotion covers it. + - name: Guard against stale promotion + id: guard + run: | + HEAD_SHA="$(git ls-remote "https://github.com/${{ github.repository }}.git" "refs/heads/${GITHUB_REF_NAME}" | cut -f1)" + if [ "$HEAD_SHA" != "${{ github.sha }}" ]; then + echo "::warning::Skipping promotion of ${{ github.sha }}: ${GITHUB_REF_NAME} has moved to ${HEAD_SHA}. Moving the deploy tags to this commit would deploy stale code; push a revert commit to roll back instead." + echo "fresh=false" >> $GITHUB_OUTPUT + else + echo "fresh=true" >> $GITHUB_OUTPUT + fi + + - name: Promote images to deploy tags + if: steps.guard.outputs.fresh == 'true' + env: + ECR_REPOS: >- + ${{ secrets.ECR_APP }} + ${{ secrets.ECR_MIGRATIONS }} + ${{ secrets.ECR_REALTIME }} + ${{ secrets.ECR_PII }} + run: | + REGISTRY="${{ steps.login-ecr.outputs.registry }}" + + if [ "${{ github.ref }}" = "refs/heads/main" ]; then + ECR_TAG="latest" + else + ECR_TAG="staging" + fi + + # Verify every sha image exists before moving any deploy tag, so a + # missing/expired image aborts the whole promotion up front. + for repo in $ECR_REPOS; do + echo "🔍 Verifying ${repo}:${{ github.sha }}" + docker buildx imagetools inspect "${REGISTRY}/${repo}:${{ github.sha }}" > /dev/null + done + + for repo in $ECR_REPOS; do + echo "🚀 Promoting ${repo}:${{ github.sha }} to ${ECR_TAG}" + docker buildx imagetools create \ + -t "${REGISTRY}/${repo}:${ECR_TAG}" \ + "${REGISTRY}/${repo}:${{ github.sha }}" + done + + # Build ARM64 images for GHCR (main branch only, runs in parallel with + # tests). Pushes only the immutable sha tag — latest-arm64/version-arm64 + # are applied by create-ghcr-manifests after the gate, so a failing run + # never moves a documented tag. build-ghcr-arm64: name: Build ARM64 (GHCR Only) - needs: [detect-version] runs-on: blacksmith-8vcpu-ubuntu-2404-arm if: github.event_name == 'push' && github.ref == 'refs/heads/main' permissions: @@ -316,21 +382,6 @@ jobs: - name: Set up Docker Buildx uses: useblacksmith/setup-docker-builder@ab5c1da94f53f5cd75c1038092aa276dddfccbba # v1 - - name: Generate ARM64 tags - id: meta - run: | - IMAGE="${{ matrix.image }}" - TAGS="${IMAGE}:latest-arm64,${IMAGE}:${{ github.sha }}-arm64" - - # Add version tag if this is a release commit - if [ "${{ needs.detect-version.outputs.is_release }}" = "true" ]; then - VERSION="${{ needs.detect-version.outputs.version }}" - TAGS="${TAGS},${IMAGE}:${VERSION}-arm64" - echo "đŸ“Ļ Adding version tag: ${VERSION}-arm64" - fi - - echo "tags=${TAGS}" >> $GITHUB_OUTPUT - - name: Build and push ARM64 to GHCR uses: useblacksmith/build-push-action@fb9e3e6a9299c78462bfadd0d93352c316adc9b8 # v2 with: @@ -338,15 +389,17 @@ jobs: file: ${{ matrix.dockerfile }} platforms: linux/arm64 push: true - tags: ${{ steps.meta.outputs.tags }} + tags: ${{ matrix.image }}:${{ github.sha }}-arm64 provenance: false sbom: false - # Create GHCR multi-arch manifests (only for main, after both builds) + # Publish all mutable GHCR tags (latest, latest-amd64/arm64, version tags) + # and the multi-arch manifests from the immutable sha tags — only on main, + # after the deploy gate (promote-images) and the ARM64 build both pass. create-ghcr-manifests: name: Create GHCR Manifests runs-on: blacksmith-2vcpu-ubuntu-2404 - needs: [build-amd64, build-ghcr-arm64, detect-version] + needs: [promote-images, build-ghcr-arm64, detect-version] if: github.event_name == 'push' && github.ref == 'refs/heads/main' permissions: packages: write @@ -366,30 +419,43 @@ jobs: username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Create and push manifests + # Same protection as promote-images: a re-run of an old run must not + # move the public latest tags back to a stale commit. Immutable tags + # (sha and version) are still published — only latest moves are gated. + - name: Guard against stale latest tags + id: guard run: | - IMAGE_BASE="${{ matrix.image }}" + HEAD_SHA="$(git ls-remote "https://github.com/${{ github.repository }}.git" "refs/heads/${GITHUB_REF_NAME}" | cut -f1)" + if [ "$HEAD_SHA" != "${{ github.sha }}" ]; then + echo "::warning::${GITHUB_REF_NAME} has moved to ${HEAD_SHA}; publishing immutable tags for ${{ github.sha }} but skipping the latest tags." + echo "fresh=false" >> $GITHUB_OUTPUT + else + echo "fresh=true" >> $GITHUB_OUTPUT + fi - # Create latest manifest - docker manifest create "${IMAGE_BASE}:latest" \ - "${IMAGE_BASE}:latest-amd64" \ - "${IMAGE_BASE}:latest-arm64" - docker manifest push "${IMAGE_BASE}:latest" + - name: Publish tags and manifests + run: | + IMAGE="${{ matrix.image }}" + SHA="${{ github.sha }}" - # Create SHA manifest - docker manifest create "${IMAGE_BASE}:${{ github.sha }}" \ - "${IMAGE_BASE}:${{ github.sha }}-amd64" \ - "${IMAGE_BASE}:${{ github.sha }}-arm64" - docker manifest push "${IMAGE_BASE}:${{ github.sha }}" + # Multi-arch manifest from the immutable per-arch sha tags + docker buildx imagetools create -t "${IMAGE}:${SHA}" \ + "${IMAGE}:${SHA}-amd64" "${IMAGE}:${SHA}-arm64" - # Create version manifest if this is a release commit if [ "${{ needs.detect-version.outputs.is_release }}" = "true" ]; then VERSION="${{ needs.detect-version.outputs.version }}" - echo "đŸ“Ļ Creating version manifest: ${VERSION}" - docker manifest create "${IMAGE_BASE}:${VERSION}" \ - "${IMAGE_BASE}:${VERSION}-amd64" \ - "${IMAGE_BASE}:${VERSION}-arm64" - docker manifest push "${IMAGE_BASE}:${VERSION}" + echo "đŸ“Ļ Publishing version tags: ${VERSION}" + docker buildx imagetools create -t "${IMAGE}:${VERSION}-amd64" "${IMAGE}:${SHA}-amd64" + docker buildx imagetools create -t "${IMAGE}:${VERSION}-arm64" "${IMAGE}:${SHA}-arm64" + docker buildx imagetools create -t "${IMAGE}:${VERSION}" \ + "${IMAGE}:${SHA}-amd64" "${IMAGE}:${SHA}-arm64" + fi + + if [ "${{ steps.guard.outputs.fresh }}" = "true" ]; then + docker buildx imagetools create -t "${IMAGE}:latest-amd64" "${IMAGE}:${SHA}-amd64" + docker buildx imagetools create -t "${IMAGE}:latest-arm64" "${IMAGE}:${SHA}-arm64" + docker buildx imagetools create -t "${IMAGE}:latest" \ + "${IMAGE}:${SHA}-amd64" "${IMAGE}:${SHA}-arm64" fi # Check if docs changed @@ -412,10 +478,10 @@ jobs: - 'apps/sim/scripts/process-docs.ts' - 'apps/sim/lib/chunkers/**' - # Process docs embeddings (only when docs change, after ECR images are pushed) + # Process docs embeddings (only when docs change, after images are promoted) process-docs: name: Process Docs - needs: [build-amd64, check-docs-changes] + needs: [promote-images, check-docs-changes] if: needs.check-docs-changes.outputs.docs_changed == 'true' uses: ./.github/workflows/docs-embeddings.yml secrets: inherit diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index 20c2239714c..5c427b3c5e6 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -9,7 +9,7 @@ permissions: jobs: test-build: - name: Test and Build + name: Lint and Test runs-on: blacksmith-8vcpu-ubuntu-2404 steps: @@ -44,14 +44,6 @@ jobs: key: ${{ github.repository }}-turbo-cache path: ./.turbo - - name: Restore Next.js build cache - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 - with: - path: ./apps/sim/.next/cache - key: ${{ runner.os }}-nextjs-${{ hashFiles('bun.lock') }} - restore-keys: | - ${{ runner.os }}-nextjs- - - name: Install dependencies run: bun install --frozen-lockfile @@ -175,6 +167,67 @@ jobs: fi echo "✅ Schema and migrations are in sync" + - name: Upload coverage to Codecov + uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5 + with: + directory: ./apps/sim/coverage + fail_ci_if_error: false + verbose: true + + # Next.js production build, in parallel with lint + tests. Sticky disks are + # cloned from the last committed snapshot per job and committed last-writer- + # wins, so concurrent mounts are safe. The bun/node_modules disks are shared + # with test-build (identical content from the same lockfile — LWW loss is + # harmless), but the Turbo cache gets its own key: with a shared key, only + # the last committer's new entries survive each run, so the test and build + # Turbo entries would evict each other nondeterministically. + build: + name: Build App + runs-on: blacksmith-8vcpu-ubuntu-2404 + + steps: + - name: Checkout code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + + - name: Setup Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.3.13 + + - name: Setup Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 + with: + node-version: latest + + - name: Mount Bun cache (Sticky Disk) + uses: useblacksmith/stickydisk@4c034ba57b706cf0e3b4b0ce098c2a3b1071580c # v1 + with: + key: ${{ github.repository }}-bun-cache + path: ~/.bun/install/cache + + - name: Mount node_modules (Sticky Disk) + uses: useblacksmith/stickydisk@4c034ba57b706cf0e3b4b0ce098c2a3b1071580c # v1 + with: + key: ${{ github.repository }}-node-modules + path: ./node_modules + + - name: Mount Turbo cache (Sticky Disk) + uses: useblacksmith/stickydisk@4c034ba57b706cf0e3b4b0ce098c2a3b1071580c # v1 + with: + key: ${{ github.repository }}-turbo-cache-build + path: ./.turbo + + - name: Restore Next.js build cache + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 + with: + path: ./apps/sim/.next/cache + key: ${{ runner.os }}-nextjs-${{ hashFiles('bun.lock') }} + restore-keys: | + ${{ runner.os }}-nextjs- + + - name: Install dependencies + run: bun install --frozen-lockfile + - name: Build application env: NODE_OPTIONS: '--no-warnings --max-old-space-size=8192' @@ -186,11 +239,4 @@ jobs: AWS_REGION: 'us-west-2' ENCRYPTION_KEY: '7cf672e460e430c1fba707575c2b0e2ad5a99dddf9b7b7e3b5646e630861db1c' # dummy key for CI only TURBO_CACHE_DIR: .turbo - run: bunx turbo run build --filter=sim - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5 - with: - directory: ./apps/sim/coverage - fail_ci_if_error: false - verbose: true \ No newline at end of file + run: bunx turbo run build --filter=sim \ No newline at end of file