ci: GitHub's runners for the builds, the fleet for what boots

.github/actionlint.yaml

@@ -1,6 +1,6 @@
 self-hosted-runner:
   labels:
-    - blacksmith-4vcpu-ubuntu-2404
-    - blacksmith-8vcpu-ubuntu-2404
-    # The self-hosted host the release report boots on: it has /dev/kvm.
-    - kvm
+    # spin-stack/ci's fleets: kvm for compat.yml, which builds and boots; bench, one instance type,
+    # for report.yml and lab.yml, whose boot times compare only with ones taken on the same host.
+    - runs-on/fleet=kvm/env=production
+    - runs-on/fleet=bench/env=production

.github/workflows/ci.yml

@@ -36,7 +36,7 @@ jobs:
   gate:
     # The check main's ruleset requires, named Gate in every spin-stack repository.
     name: Gate
-    runs-on: blacksmith-8vcpu-ubuntu-2404
+    runs-on: ubuntu-latest
     # A ceiling, not a budget. The default is six hours, and a `go test` that deadlocks
     # would hold a runner for all of them without printing a failure.
     timeout-minutes: 15

.github/workflows/compat.yml

@@ -5,20 +5,19 @@ name: Compat
 # change on main to what a resumed guest is loaded into - the machine's definition, QEMU, the
 # kernel, the firmware and their pins - and before every release.
 #
-# On the KVM runner, because it boots: the newest release is unpacked as the machine before, and
-# the machine here is that tree with this change's QEMU, firmware and kernel built over it - the
-# base image is the release's, since a checkpoint resumes with the disks it was saved with. The
-# builds read and write a BuildKit cache that outlives the run (checkout keeps .cache/), so a
-# change that leaves QEMU and the kernel alone rebuilds neither.
+# On the KVM fleet (spin-stack/ci), because it boots: the newest release is unpacked as the machine
+# before, and the machine here is that tree with this change's QEMU, firmware and kernel built over
+# it - the base image is the release's, since a checkpoint resumes with the disks it was saved
+# with. The builds read the BuildKit cache main's lanes export (CACHE_BACKEND=gha), so a change
+# that leaves QEMU and the kernel alone compiles neither.
 #
 # The test holds the promise only where it is made: when the two fingerprints differ the contract
 # moved on purpose, the test says so and passes, and the release notes say it for the fleet.
 #
-# Not on a pull request: the KVM runner's job hook refuses that event, since a workflow asking
-# for its runners would run a pull request's code as root there. It runs on main after a merge
-# that touches the machine, on a dispatch - a branch tried before its pull request merges - and
-# as a job release.yml waits for, so no release is published that a checkpoint of the one
-# before it does not resume on.
+# Not on a pull request: it is main's and a release's lane, and a branch is tried by dispatch
+# before its pull request merges. It runs on main after a merge that touches the machine, and as
+# a job release.yml waits for, so no release is published that a checkpoint of the one before it
+# does not resume on.
 
 on:
   push:
@@ -46,29 +45,36 @@ concurrency:
 
 jobs:
   compat:
-    runs-on: [self-hosted, linux, x64, kvm]
+    runs-on: runs-on/fleet=kvm/env=production
     timeout-minutes: 120
     env:
       GH_TOKEN: ${{ github.token }}
     steps:
       - name: Checkout
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
-        with:
-          # .cache/buildkit is the runner's QEMU and kernel cache; a clean checkout would empty it.
-          clean: false
 
+      # Go's caches come back from the fleet's cache (its S3 bucket): the instance starts with
+      # nothing.
       - name: Set up Go
         uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
         with:
           go-version-file: go.mod
-          cache: false
 
-      # A builder that can export its cache to the directory above; the runner keeps it.
-      - name: A BuildKit that keeps its cache
+      # A builder that can read a cache Docker's own cannot.
+      - name: A BuildKit that reads the lanes' cache
         run: go tool task boot:buildkit
 
+      # The tokens `type=gha` needs in a `run:` step; qemu.yml says why.
+      - uses: crazy-max/ghaction-github-runtime@04d248b84655b509d8c44dc1d6f990c879747487 # v4.0.0
+
+      # The guests' vsock transport, loaded and made this user's: a module on the fleet's kernel
+      # that nothing loads before the first QEMU opens /dev/vhost-vsock.
+      - name: Load vhost_vsock
+        run: go tool task vsock
+
+      # Reads the lanes' scopes and writes none back: main's lanes keep them current.
       - name: Build this change's QEMU and kernel, and the tools
-        run: go tool task qemu:build kernel:build tools
+        run: go tool task qemu:build kernel:build tools CACHE_BACKEND=gha CACHE_EXPORT=false
 
       # The newest release, checked against its sha256 and its signature, as the machine before;
       # this change's machine is the same tree with what was just built in place of its own. The

.github/workflows/image.yml

@@ -40,7 +40,7 @@ concurrency:
 jobs:
   # Whether this change can reach e2fsprogs or the image; see the note in qemu.yml.
   changed:
-    runs-on: blacksmith-8vcpu-ubuntu-2404
+    runs-on: ubuntu-latest
     timeout-minutes: 10
     outputs:
       build: ${{ steps.touches.outputs.build }}
@@ -65,7 +65,7 @@ jobs:
   build:
     needs: changed
     if: needs.changed.outputs.build == 'true'
-    runs-on: blacksmith-8vcpu-ubuntu-2404
+    runs-on: ubuntu-latest
     timeout-minutes: 60
     steps:
       - name: Checkout
@@ -78,13 +78,10 @@ jobs:
           go-version-file: go.mod
           cache: true
 
-      # One sticky disk per thing built, and no silent fallback; see the note in qemu.yml. This
-      # job builds e2fsprogs and the image builder, which is one job and so one disk.
-      - name: Setup Blacksmith Builder
-        uses: useblacksmith/setup-docker-builder@19215110ab936351210feebdfa5b440b4493e184 # v2.2.0
-        with:
-          cache-key: spin-machine/image
-          nofallback: true
+      # A builder of its own (the docker-container driver): Docker's default one cannot export its
+      # cache. What it reads and writes is GitHub's cache, in the scopes Taskfile.yml decides
+      # (CACHE_BACKEND=gha), so the runner itself keeps nothing between runs.
+      - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
 
       # Without this, `--cache-to type=gha` below is a silent no-op: the tokens BuildKit
       # needs reach JavaScript actions and not `run:` steps. See the long note in qemu.yml.

.github/workflows/kernel.yml

@@ -37,7 +37,7 @@ concurrency:
 jobs:
   # Whether this change can reach the kernel; see the note in qemu.yml.
   changed:
-    runs-on: blacksmith-8vcpu-ubuntu-2404
+    runs-on: ubuntu-latest
     timeout-minutes: 10
     outputs:
       build: ${{ steps.touches.outputs.build }}
@@ -62,7 +62,7 @@ jobs:
   build:
     needs: changed
     if: needs.changed.outputs.build == 'true'
-    runs-on: blacksmith-8vcpu-ubuntu-2404
+    runs-on: ubuntu-latest
     timeout-minutes: 180
     steps:
       - name: Checkout
@@ -75,12 +75,10 @@ jobs:
           go-version-file: go.mod
           cache: true
 
-      # One sticky disk per thing built, and no silent fallback; see the note in qemu.yml.
-      - name: Setup Blacksmith Builder
-        uses: useblacksmith/setup-docker-builder@19215110ab936351210feebdfa5b440b4493e184 # v2.2.0
-        with:
-          cache-key: spin-machine/kernel
-          nofallback: true
+      # A builder of its own (the docker-container driver): Docker's default one cannot export its
+      # cache. What it reads and writes is GitHub's cache, in the scopes Taskfile.yml decides
+      # (CACHE_BACKEND=gha), so the runner itself keeps nothing between runs.
+      - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
 
       # Without this, `--cache-to type=gha` below is a silent no-op: the tokens BuildKit
       # needs reach JavaScript actions and not `run:` steps. See the long note in qemu.yml.

.github/workflows/lab.yml

@@ -1,7 +1,7 @@
 name: Lab
 
-# One of the boot/ experiments, run on the KVM runner against a published release: the same Task
-# targets a developer runs, on a host that is doing nothing else.
+# One of the boot/ experiments, run on the bench fleet (spin-stack/ci, one instance type) against
+# a published release: the same Task targets a developer runs, on a host that is doing nothing else.
 #
 # A workflow and not a laptop because a boot time is only worth comparing on a quiet host, and a
 # developer's machine is not one: three runs of one kernel on one read 59.5, 96.8 and 167.7 ms
@@ -73,13 +73,13 @@ permissions:
   contents: read
   actions: read
 
-# No concurrency group: the runner takes one job at a time and queues the rest, and a group
+# No concurrency group: the bench fleet runs one job at a time and queues the rest, and a group
 # keeps one run pending and cancels the one before it, which lost two of three experiments
 # queued back to back.
 
 jobs:
   lab:
-    runs-on: [self-hosted, linux, x64, kvm]
+    runs-on: runs-on/fleet=bench/env=production
     timeout-minutes: 360
     env:
       GH_TOKEN: ${{ github.token }}
@@ -96,13 +96,17 @@ jobs:
       - name: Checkout
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
 
-      # No Actions cache: this runner keeps Go's module and build caches on its own disk, and
-      # the cache service only stored another copy per branch, against the repository's limit.
+      # Go's caches come back from the fleet's cache (its S3 bucket): the instance starts with
+      # nothing.
       - name: Set up Go
         uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
         with:
           go-version-file: go.mod
-          cache: false
+
+      # The guests' vsock transport, loaded and made this user's: a module on the fleet's kernel
+      # that nothing loads before the first QEMU opens /dev/vhost-vsock.
+      - name: Load vhost_vsock
+        run: go tool task vsock
 
       - name: Unpack the release
         run: go tool task lab:unpack

.github/workflows/qemu.yml

@@ -46,7 +46,7 @@ jobs:
   # Whether this change can reach QEMU at all; see the note on the triggers. Skipped, the build
   # job reports as skipped, which a required check counts as passed.
   changed:
-    runs-on: blacksmith-8vcpu-ubuntu-2404
+    runs-on: ubuntu-latest
     timeout-minutes: 10
     outputs:
       build: ${{ steps.touches.outputs.build }}
@@ -72,7 +72,7 @@ jobs:
   build:
     needs: changed
     if: needs.changed.outputs.build == 'true'
-    runs-on: blacksmith-8vcpu-ubuntu-2404
+    runs-on: ubuntu-latest
     # A cold build is tens of minutes and a warm one is a cache restore, so the spread is
     # wide and the ceiling has to sit above the cold case. What it is really for is the
     # other end: a `configure` that stops on a prompt would hold a runner for six hours
@@ -90,22 +90,10 @@ jobs:
           go-version-file: go.mod
           cache: true
 
-      # cache-key is required as of v2 and names the sticky disk the builder keeps its state
-      # on, so it is one key per thing built, for the same reason the BuildKit scopes in
-      # Taskfile.yml are: share one and each build is handed the last one's disk and starts by
-      # discarding it.
-      #
-      # nofallback, because the default is to degrade. A builder that cannot be set up
-      # otherwise reports the error, falls back to a local one and carries on, so the run goes
-      # green having lost the disk — which is how v2's new required input arrived unnoticed.
-      # The cost of this is that an outage at a third party turns the lane red instead of
-      # making it slow, which is the trade this repository makes everywhere else: a build that
-      # did not do what it says is worth more as a failure than as a warning nobody reads.
-      - name: Setup Blacksmith Builder
-        uses: useblacksmith/setup-docker-builder@19215110ab936351210feebdfa5b440b4493e184 # v2.2.0
-        with:
-          cache-key: spin-machine/qemu
-          nofallback: true
+      # A builder of its own (the docker-container driver): Docker's default one cannot export its
+      # cache. What it reads and writes is GitHub's cache, in the scopes Taskfile.yml decides
+      # (CACHE_BACKEND=gha), so the runner itself keeps nothing between runs.
+      - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
 
       # What makes `--cache-to type=gha` do anything at all.
       #
@@ -190,8 +178,8 @@ jobs:
       # The runtime image, tagged with the pinned QEMU version: what `task qemu:fetch` pulls,
       # so CI's argument check runs main's QEMU in seconds instead of compiling it. The
       # binaries are static, so the image around them carries nothing they need. It reads the
-      # builder stage the step above compiled from the `qemu` scope, and writes no cache: the
-      # runner's sticky disk already holds these layers.
+      # builder stage the step above compiled from the `qemu` scope, and writes no cache: that
+      # scope holds these layers already.
       - name: Publish the runtime image
         if: github.ref == 'refs/heads/main' && github.event_name != 'pull_request'
         run: go tool task qemu:publish

.github/workflows/release.yml

@@ -50,7 +50,7 @@ jobs:
   release:
     needs: compat
     if: github.ref == 'refs/heads/main'
-    runs-on: blacksmith-8vcpu-ubuntu-2404
+    runs-on: ubuntu-latest
     # A warm run reads main's BuildKit cache and takes a few minutes (2m13s, 2026-09-30). The
     # ceiling is for the cold one, which compiles QEMU and the kernel, each tens of minutes.
     timeout-minutes: 180
@@ -74,19 +74,13 @@ jobs:
           go-version-file: go.mod
           cache: true
 
-      # Its own sticky disk, and not one of the lanes'. This job builds all four artefacts, so
-      # it would want every lane's disk at once and can only have one — and a release can
-      # overlap a lane, which sharing a key would make them contend for. The BuildKit scopes it
-      # reads are still the lanes', so what main already compiled is not compiled again; see
-      # the note in qemu.yml and CACHE_BACKEND in Taskfile.yml.
-      # And no silent fallback, which matters most here: a release that compiled QEMU and the
-      # kernel from cold because its builder quietly became a local one is hours, and it would
-      # look like nothing but a slow release.
-      - name: Setup Blacksmith Builder
-        uses: useblacksmith/setup-docker-builder@19215110ab936351210feebdfa5b440b4493e184 # v2.2.0
-        with:
-          cache-key: spin-machine/release
-          nofallback: true
+      # What main's lanes already compiled is read, not compiled again: the BuildKit scopes are the
+      # lanes' (CACHE_BACKEND in Taskfile.yml); a cold release compiles QEMU and the kernel, each
+      # tens of minutes, which the ceiling above is for.
+      # A builder of its own (the docker-container driver): Docker's default one cannot export its
+      # cache. What it reads and writes is GitHub's cache, in the scopes Taskfile.yml decides
+      # (CACHE_BACKEND=gha), so the runner itself keeps nothing between runs.
+      - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
 
       # Without this, `--cache-to type=gha` below is a silent no-op: the tokens BuildKit
       # needs reach JavaScript actions and not `run:` steps. See the long note in qemu.yml.

.github/workflows/report.yml

@@ -4,11 +4,12 @@ name: Report
 # through the feature matrix (boot/report_test.go), written as report.json, and compared with the
 # newest earlier release that has one.
 #
-# A workflow of its own because it boots: a hosted runner has no /dev/kvm for it, and a time
-# measured under emulation is not a time anybody's machine takes. The runner is one host, on
-# purpose - boot times compare only between reports taken on the same kind of host, and compare
-# says so when they were not. It needs /dev/kvm and /dev/vhost-vsock usable by its user, sudo -n
-# for modprobe nbd and qemu-nbd, and gh (docs/releasing.md).
+# A workflow of its own because it boots: GitHub's runners have no /dev/kvm for it, and a time
+# measured under emulation is not a time anybody's machine takes. The runner is one instance type,
+# on purpose - the bench fleet of spin-stack/ci, a c8i.4xlarge with nested virtualization - since
+# boot times compare only between reports taken on the same kind of host, and compare says so
+# when they were not. It needs /dev/kvm and /dev/vhost-vsock usable by its user, sudo -n for
+# modprobe nbd and qemu-nbd, and gh (docs/releasing.md).
 #
 # release.yml calls it with publish, which uploads report.json to the release and appends the
 # comparison to its notes. Run by hand it measures any release - to check the runner, or to see
@@ -36,7 +37,7 @@ permissions:
 
 jobs:
   report:
-    runs-on: [self-hosted, linux, x64, kvm]
+    runs-on: runs-on/fleet=bench/env=production
     timeout-minutes: 360
     env:
       GH_TOKEN: ${{ github.token }}
@@ -52,13 +53,17 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
-      # No Actions cache: this runner keeps Go's module and build caches on its own disk, and
-      # the cache service only stored another copy per branch, against the repository's limit.
+      # Go's caches come back from the fleet's cache (its S3 bucket): the instance starts with
+      # nothing.
       - name: Set up Go
         uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
         with:
           go-version-file: go.mod
-          cache: false
+
+      # The guests' vsock transport, loaded and made this user's: a module on the fleet's kernel
+      # that nothing loads before the first QEMU opens /dev/vhost-vsock.
+      - name: Load vhost_vsock
+        run: go tool task vsock
 
       # What this runner can do, said before anything is measured: a row the report skips is one
       # this host lacks the device for, and the summary says which before the report does.

AGENTS.md

@@ -90,10 +90,10 @@ not silently become production behaviour.
   CPU affinity), document why it does not.
 - Remove or quarantine an experiment once it no longer has an active question. Keep the
   conclusion and durable evidence, not a permanent switch in the release path.
-- A measurement that decides something is taken on the KVM runner (construct), through
-  `.github/workflows/lab.yml`, one run at a time: its three runners share one machine, and two
-  lab runs side by side measure each other. A laptop has neither the Spin OS kernel nor a quiet
-  host, so its numbers are leads, not findings. Compare with enough boots to beat the noise -
+- A measurement that decides something is taken on the bench fleet (spin-stack/ci, one instance
+  type) through `.github/workflows/lab.yml`, one run at a time: two lab runs side by side measure
+  each other. A laptop is not a quiet host, so its numbers are leads, not findings, and a number
+  taken on another kind of host compares with nothing. Compare with enough boots to beat the noise -
   three a row moved a p50 by 5-11% between runs of one release (#81) - and alternate the
   variants rather than running them in blocks.
 - What the guest's memory and disk were measured to cost, and why each default is what it is,