From 8fa7f0cc4e143fab198c667fb207ba89fe8117e2 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 28 Aug 2026 01:46:08 -0700 Subject: [PATCH] improvement(ci): bound docker layer caches and right-size ten runners MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Blacksmith sticky disks backing our docker layer caches had no eviction policy. setup-docker-builder skips pruning entirely unless max-cache-size-mb is set, and BuildKit's own GC is time-based only (8 days unused), so on a repo that builds this often nothing ever aged out: app.Dockerfile/linux-amd64 hit 351 GB within a day of being created, and realtime — an image under 300 MB — sat at 249 GB. Layer caches alone were 920 GB across ten disks. Cap them per image via a cache_mb matrix field, alongside the bs_runner field that already encodes per-image sizing. The app image keeps 100 GB (several generations over its working set of ~34 layers plus monorepo apt/bun cache mounts); everything else takes the 25 GB default, which is still 4x the tightest working set in the matrix (pii, whose spaCy models are ~2.2 GB). The fallback lives in the composite action rather than an input default, because an unset matrix key arrives as the empty string and would bypass a default — silently restoring unbounded growth on any row that forgot the field. Runner sizes follow measured CPU and memory percentiles over 30 days: - CodeQL splits per language. javascript-typescript peaks at 19.5 GB so it stays on 8 vCPU; actions peaks at 1.3 GB and averages 22% CPU over a 39s median run, and drops to 4 vCPU. - The pii and realtime image builds drop to 2 vCPU. Both already ran on 8 vCPU earlier in the window, so the 8->4 step is measured, not modelled: realtime went 52s -> 51s and pii 24s -> 27s. - Five desktop release jobs drop to 2 vCPU. They peak under 0.4 GB and finish in 4-13s. None of these sit on their group's critical path — each has 190-275s of slack behind an app build that dominates it — so wall-clock is unchanged. Those five desktop jobs also hardcoded a Blacksmith label with no CI_PROVIDER fallback, against the invariant stated at the top of ci.yml. In GitHub break-glass mode they would have sat in `queued` forever; they now fall back like every other job. Left alone deliberately: the 16 vCPU app builds (memory-bound, and 16 vCPU measured 2.1x faster and 6% cheaper than 8 vCPU), Lint and Test (CPU-bound, 62% of the run above 80%), and the push-path Build App (build-amd64 has no needs:, so it is what stops a migration applying for a build that cannot ship). --- .github/actions/docker-build/action.yml | 20 ++++++++++++++++ .github/workflows/ci.yml | 31 ++++++++++++++++++------- .github/workflows/codeql.yml | 13 +++++++++-- 3 files changed, 53 insertions(+), 11 deletions(-) diff --git a/.github/actions/docker-build/action.yml b/.github/actions/docker-build/action.yml index a607a720afd..b8727501f52 100644 --- a/.github/actions/docker-build/action.yml +++ b/.github/actions/docker-build/action.yml @@ -19,6 +19,13 @@ inputs: tags: description: Comma-separated list of tags to push. required: true + max-cache-size-mb: + description: >- + Layer cache to retain after the post-job prune, in MB. Must stay above one + build's working set (base + dependency layers + RUN --mount=type=cache + dirs) or every build evicts what the next one needs. Falls back to the + small-image default below when empty. + required: false # Registry logins must precede this action. provenance/sbom stay off: attestation # manifests break `imagetools create` retagging in promote-images. @@ -42,11 +49,24 @@ runs: PLATFORMS: ${{ inputs.platforms }} run: echo "value=${GITHUB_REPOSITORY##*/}/${FILE#./}/${PLATFORMS//\//-}" >> "$GITHUB_OUTPUT" + # max-cache-size-mb is what bounds the disk: BuildKit's default GC is + # time-based only (layers unused for 8 days), and setup-docker-builder skips + # pruning altogether when the value is empty. On a repo that builds this + # often nothing ever ages out, so the disks grew without limit — + # app.Dockerfile/linux-amd64 reached 351 GB inside a day, and realtime, whose + # image is under 300 MB, sat at 249 GB. Sticky disks bill at ~$0.51/GB-month, + # so that was real money for layers no build would ever read again. + # + # The fallback is here rather than an input `default:` because callers pass + # this from a matrix field, and an unset matrix key arrives as the empty + # string — which counts as "provided", so a `default:` would never apply and + # a row that forgot the field would silently go back to unbounded growth. - name: Set up Blacksmith builder if: inputs.provider == '' || inputs.provider == 'blacksmith' uses: useblacksmith/setup-docker-builder@a5256a73e30f09e37e3eceb8ca36043d17621d24 # v2 with: cache-key: ${{ steps.cache-key.outputs.value }} + max-cache-size-mb: ${{ inputs.max-cache-size-mb || '25600' }} - name: Build and push (Blacksmith) if: inputs.provider == '' || inputs.provider == 'blacksmith' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 746c341cae8..aa1be99ea97 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -76,7 +76,7 @@ jobs: # (/api/desktop/update) starts offering automatically. detect-desktop-changes: name: Detect Desktop Changes - runs-on: blacksmith-4vcpu-ubuntu-2404 + runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} timeout-minutes: 5 if: github.event_name == 'push' && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/staging') outputs: @@ -165,7 +165,15 @@ jobs: # build` ~260s). The same `next build` runs on 16 vCPU in the separate # Build App verification job, which does not gate anything; this one # was doing comparable work on half the cores. + # + # cache_mb is the layer cache the post-job prune retains, and it is the + # only reason the sticky disks stay bounded — see docker-build's + # action.yml. Rows that omit it take the small-image default there. The + # app image overrides because it carries ~34 layers plus apt and bun + # cache mounts for the whole monorepo; 100 GB is several builds' worth + # of headroom over that working set. - dockerfile: ./docker/app.Dockerfile + cache_mb: '102400' ecr_repo_secret: ECR_APP gh_runner: linux-x64-8-core bs_runner: blacksmith-16vcpu-ubuntu-2404 @@ -176,11 +184,11 @@ jobs: - dockerfile: ./docker/realtime.Dockerfile ecr_repo_secret: ECR_REALTIME gh_runner: ubuntu-latest - bs_runner: blacksmith-4vcpu-ubuntu-2404 + bs_runner: blacksmith-2vcpu-ubuntu-2404 - dockerfile: ./docker/pii.Dockerfile ecr_repo_secret: ECR_PII gh_runner: ubuntu-latest - bs_runner: blacksmith-4vcpu-ubuntu-2404 + bs_runner: blacksmith-2vcpu-ubuntu-2404 steps: - name: Checkout code uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 @@ -214,6 +222,7 @@ jobs: file: ${{ matrix.dockerfile }} platforms: linux/amd64 tags: ${{ steps.login-ecr.outputs.registry }}/${{ steps.ecr-repo.outputs.name }}:dev + max-cache-size-mb: ${{ matrix.cache_mb }} # Dev: deploy Trigger.dev background tasks to the preview "dev-sim" branch. # Gated after migrate-dev for the same reason as build-dev — the new task @@ -280,6 +289,7 @@ jobs: matrix: include: - dockerfile: ./docker/app.Dockerfile + cache_mb: '102400' ghcr_image: ghcr.io/simstudioai/simstudio ecr_repo_secret: ECR_APP gh_runner: linux-x64-8-core @@ -293,12 +303,12 @@ jobs: ghcr_image: ghcr.io/simstudioai/realtime ecr_repo_secret: ECR_REALTIME gh_runner: ubuntu-latest - bs_runner: blacksmith-4vcpu-ubuntu-2404 + bs_runner: blacksmith-2vcpu-ubuntu-2404 - dockerfile: ./docker/pii.Dockerfile ghcr_image: ghcr.io/simstudioai/pii ecr_repo_secret: ECR_PII gh_runner: ubuntu-latest - bs_runner: blacksmith-4vcpu-ubuntu-2404 + bs_runner: blacksmith-2vcpu-ubuntu-2404 # No ECR repo is provisioned for cron, so it publishes to GHCR only. # The tag step below omits the ECR tag when the repo name is empty. - dockerfile: ./docker/cron.Dockerfile @@ -382,6 +392,7 @@ jobs: file: ${{ matrix.dockerfile }} platforms: linux/amd64 tags: ${{ steps.meta.outputs.tags }} + max-cache-size-mb: ${{ matrix.cache_mb }} # Promote the sha-tagged ECR images to the deploy tags once tests and # migrations pass. Pushing the ECR latest/staging tag is what triggers @@ -484,6 +495,7 @@ jobs: # hang a release in `queued` rather than fail a PR. include: - dockerfile: ./docker/app.Dockerfile + cache_mb: '102400' image: ghcr.io/simstudioai/simstudio gh_runner: linux-arm64-8-core bs_runner: blacksmith-8vcpu-ubuntu-2404-arm @@ -522,6 +534,7 @@ jobs: file: ${{ matrix.dockerfile }} platforms: linux/arm64 tags: ${{ matrix.image }}:${{ github.sha }}-arm64 + max-cache-size-mb: ${{ matrix.cache_mb }} # Publish all mutable GHCR tags (latest, latest-amd64/arm64, version tags) # and the multi-arch manifests from the immutable sha tags — only on main, @@ -675,7 +688,7 @@ jobs: # Job-level `if:` cannot read the secrets context, hence the probe job. check-desktop-signing: name: Check Desktop Signing Secrets - runs-on: blacksmith-4vcpu-ubuntu-2404 + runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} timeout-minutes: 2 needs: [detect-version, detect-desktop-changes] # !cancelled(): detect-desktop-changes is skipped on main (and @@ -724,7 +737,7 @@ jobs: # remains testable end to end with a manual download. create-desktop-prerelease: name: Create Desktop Prerelease - runs-on: blacksmith-4vcpu-ubuntu-2404 + runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} timeout-minutes: 5 needs: [detect-desktop-changes, check-desktop-signing] # Requires the signing probe to have actually succeeded (not just "not @@ -813,7 +826,7 @@ jobs: # point of view. publish-desktop-prerelease: name: Publish Desktop Prerelease - runs-on: blacksmith-4vcpu-ubuntu-2404 + runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} timeout-minutes: 5 needs: [create-desktop-prerelease, desktop-prerelease] permissions: @@ -837,7 +850,7 @@ jobs: # are always garbage by this point — the current run's release is published. prune-desktop-prereleases: name: Prune Desktop Prereleases - runs-on: blacksmith-4vcpu-ubuntu-2404 + runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} timeout-minutes: 5 needs: [publish-desktop-prerelease] permissions: diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index f44ca430c7c..1e7e99ce165 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -54,7 +54,12 @@ permissions: jobs: analyze: name: Analyze ${{ matrix.language }} - runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-8vcpu-ubuntu-2404' || 'ubuntu-latest' }} + # Sized per language, not per workflow. The two analyses are nothing alike: + # javascript-typescript peaks at 19.5 GB (p95 over 3090 runs), so it needs + # the 8 vCPU tier's 30.4 GB and would OOM on the 4 vCPU tier's 15.2 GB; the + # actions analysis peaks at 1.3 GB and averages 22% CPU over a 39s median + # run, so 8 vCPU was 4x more machine than it ever used. + runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && matrix.bs_runner || 'ubuntu-latest' }} timeout-minutes: 60 if: github.event.pull_request.draft != true permissions: @@ -71,7 +76,11 @@ jobs: # entries default setup listed were one analysis, not three. # `javascript-typescript` is the documented spelling. Python dropped: # 7 files in the tree. - language: [javascript-typescript, actions] + include: + - language: javascript-typescript + bs_runner: blacksmith-8vcpu-ubuntu-2404 + - language: actions + bs_runner: blacksmith-4vcpu-ubuntu-2404 steps: - name: Checkout repository