diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index 21de2a0..f7423cd 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -1,6 +1,6 @@ self-hosted-runner: labels: - - blacksmith-4vcpu-ubuntu-2404 - - blacksmith-8vcpu-ubuntu-2404 - # The self-hosted host the release report boots on: it has /dev/kvm. - - kvm + # spin-stack/ci's fleets: kvm for compat.yml, which builds and boots; bench, one instance type, + # for report.yml and lab.yml, whose boot times compare only with ones taken on the same host. + - runs-on/fleet=kvm/env=production + - runs-on/fleet=bench/env=production diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8cc21ab..070dd37 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,7 +36,7 @@ jobs: gate: # The check main's ruleset requires, named Gate in every spin-stack repository. name: Gate - runs-on: blacksmith-8vcpu-ubuntu-2404 + runs-on: ubuntu-latest # A ceiling, not a budget. The default is six hours, and a `go test` that deadlocks # would hold a runner for all of them without printing a failure. timeout-minutes: 15 diff --git a/.github/workflows/compat.yml b/.github/workflows/compat.yml index 9b6f12b..cf12637 100644 --- a/.github/workflows/compat.yml +++ b/.github/workflows/compat.yml @@ -5,20 +5,19 @@ name: Compat # change on main to what a resumed guest is loaded into - the machine's definition, QEMU, the # kernel, the firmware and their pins - and before every release. # -# On the KVM runner, because it boots: the newest release is unpacked as the machine before, and -# the machine here is that tree with this change's QEMU, firmware and kernel built over it - the -# base image is the release's, since a checkpoint resumes with the disks it was saved with. The -# builds read and write a BuildKit cache that outlives the run (checkout keeps .cache/), so a -# change that leaves QEMU and the kernel alone rebuilds neither. +# On the KVM fleet (spin-stack/ci), because it boots: the newest release is unpacked as the machine +# before, and the machine here is that tree with this change's QEMU, firmware and kernel built over +# it - the base image is the release's, since a checkpoint resumes with the disks it was saved +# with. The builds read the BuildKit cache main's lanes export (CACHE_BACKEND=gha), so a change +# that leaves QEMU and the kernel alone compiles neither. # # The test holds the promise only where it is made: when the two fingerprints differ the contract # moved on purpose, the test says so and passes, and the release notes say it for the fleet. # -# Not on a pull request: the KVM runner's job hook refuses that event, since a workflow asking -# for its runners would run a pull request's code as root there. It runs on main after a merge -# that touches the machine, on a dispatch - a branch tried before its pull request merges - and -# as a job release.yml waits for, so no release is published that a checkpoint of the one -# before it does not resume on. +# Not on a pull request: it is main's and a release's lane, and a branch is tried by dispatch +# before its pull request merges. It runs on main after a merge that touches the machine, and as +# a job release.yml waits for, so no release is published that a checkpoint of the one before it +# does not resume on. on: push: @@ -46,29 +45,36 @@ concurrency: jobs: compat: - runs-on: [self-hosted, linux, x64, kvm] + runs-on: runs-on/fleet=kvm/env=production timeout-minutes: 120 env: GH_TOKEN: ${{ github.token }} steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - # .cache/buildkit is the runner's QEMU and kernel cache; a clean checkout would empty it. - clean: false + # Go's caches come back from the fleet's cache (its S3 bucket): the instance starts with + # nothing. - name: Set up Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version-file: go.mod - cache: false - # A builder that can export its cache to the directory above; the runner keeps it. - - name: A BuildKit that keeps its cache + # A builder that can read a cache Docker's own cannot. + - name: A BuildKit that reads the lanes' cache run: go tool task boot:buildkit + # The tokens `type=gha` needs in a `run:` step; qemu.yml says why. + - uses: crazy-max/ghaction-github-runtime@04d248b84655b509d8c44dc1d6f990c879747487 # v4.0.0 + + # The guests' vsock transport, loaded and made this user's: a module on the fleet's kernel + # that nothing loads before the first QEMU opens /dev/vhost-vsock. + - name: Load vhost_vsock + run: go tool task vsock + + # Reads the lanes' scopes and writes none back: main's lanes keep them current. - name: Build this change's QEMU and kernel, and the tools - run: go tool task qemu:build kernel:build tools + run: go tool task qemu:build kernel:build tools CACHE_BACKEND=gha CACHE_EXPORT=false # The newest release, checked against its sha256 and its signature, as the machine before; # this change's machine is the same tree with what was just built in place of its own. The diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml index d3d7ca6..0bbe8b3 100644 --- a/.github/workflows/image.yml +++ b/.github/workflows/image.yml @@ -40,7 +40,7 @@ concurrency: jobs: # Whether this change can reach e2fsprogs or the image; see the note in qemu.yml. changed: - runs-on: blacksmith-8vcpu-ubuntu-2404 + runs-on: ubuntu-latest timeout-minutes: 10 outputs: build: ${{ steps.touches.outputs.build }} @@ -65,7 +65,7 @@ jobs: build: needs: changed if: needs.changed.outputs.build == 'true' - runs-on: blacksmith-8vcpu-ubuntu-2404 + runs-on: ubuntu-latest timeout-minutes: 60 steps: - name: Checkout @@ -78,13 +78,10 @@ jobs: go-version-file: go.mod cache: true - # One sticky disk per thing built, and no silent fallback; see the note in qemu.yml. This - # job builds e2fsprogs and the image builder, which is one job and so one disk. - - name: Setup Blacksmith Builder - uses: useblacksmith/setup-docker-builder@19215110ab936351210feebdfa5b440b4493e184 # v2.2.0 - with: - cache-key: spin-machine/image - nofallback: true + # A builder of its own (the docker-container driver): Docker's default one cannot export its + # cache. What it reads and writes is GitHub's cache, in the scopes Taskfile.yml decides + # (CACHE_BACKEND=gha), so the runner itself keeps nothing between runs. + - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 # Without this, `--cache-to type=gha` below is a silent no-op: the tokens BuildKit # needs reach JavaScript actions and not `run:` steps. See the long note in qemu.yml. diff --git a/.github/workflows/kernel.yml b/.github/workflows/kernel.yml index 65d1ed9..15f6038 100644 --- a/.github/workflows/kernel.yml +++ b/.github/workflows/kernel.yml @@ -37,7 +37,7 @@ concurrency: jobs: # Whether this change can reach the kernel; see the note in qemu.yml. changed: - runs-on: blacksmith-8vcpu-ubuntu-2404 + runs-on: ubuntu-latest timeout-minutes: 10 outputs: build: ${{ steps.touches.outputs.build }} @@ -62,7 +62,7 @@ jobs: build: needs: changed if: needs.changed.outputs.build == 'true' - runs-on: blacksmith-8vcpu-ubuntu-2404 + runs-on: ubuntu-latest timeout-minutes: 180 steps: - name: Checkout @@ -75,12 +75,10 @@ jobs: go-version-file: go.mod cache: true - # One sticky disk per thing built, and no silent fallback; see the note in qemu.yml. - - name: Setup Blacksmith Builder - uses: useblacksmith/setup-docker-builder@19215110ab936351210feebdfa5b440b4493e184 # v2.2.0 - with: - cache-key: spin-machine/kernel - nofallback: true + # A builder of its own (the docker-container driver): Docker's default one cannot export its + # cache. What it reads and writes is GitHub's cache, in the scopes Taskfile.yml decides + # (CACHE_BACKEND=gha), so the runner itself keeps nothing between runs. + - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 # Without this, `--cache-to type=gha` below is a silent no-op: the tokens BuildKit # needs reach JavaScript actions and not `run:` steps. See the long note in qemu.yml. diff --git a/.github/workflows/lab.yml b/.github/workflows/lab.yml index 05cbaa6..2a968f0 100644 --- a/.github/workflows/lab.yml +++ b/.github/workflows/lab.yml @@ -1,7 +1,7 @@ name: Lab -# One of the boot/ experiments, run on the KVM runner against a published release: the same Task -# targets a developer runs, on a host that is doing nothing else. +# One of the boot/ experiments, run on the bench fleet (spin-stack/ci, one instance type) against +# a published release: the same Task targets a developer runs, on a host that is doing nothing else. # # A workflow and not a laptop because a boot time is only worth comparing on a quiet host, and a # developer's machine is not one: three runs of one kernel on one read 59.5, 96.8 and 167.7 ms @@ -73,13 +73,13 @@ permissions: contents: read actions: read -# No concurrency group: the runner takes one job at a time and queues the rest, and a group +# No concurrency group: the bench fleet runs one job at a time and queues the rest, and a group # keeps one run pending and cancels the one before it, which lost two of three experiments # queued back to back. jobs: lab: - runs-on: [self-hosted, linux, x64, kvm] + runs-on: runs-on/fleet=bench/env=production timeout-minutes: 360 env: GH_TOKEN: ${{ github.token }} @@ -96,13 +96,17 @@ jobs: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - # No Actions cache: this runner keeps Go's module and build caches on its own disk, and - # the cache service only stored another copy per branch, against the repository's limit. + # Go's caches come back from the fleet's cache (its S3 bucket): the instance starts with + # nothing. - name: Set up Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version-file: go.mod - cache: false + + # The guests' vsock transport, loaded and made this user's: a module on the fleet's kernel + # that nothing loads before the first QEMU opens /dev/vhost-vsock. + - name: Load vhost_vsock + run: go tool task vsock - name: Unpack the release run: go tool task lab:unpack diff --git a/.github/workflows/qemu.yml b/.github/workflows/qemu.yml index b6327ce..4ab5400 100644 --- a/.github/workflows/qemu.yml +++ b/.github/workflows/qemu.yml @@ -46,7 +46,7 @@ jobs: # Whether this change can reach QEMU at all; see the note on the triggers. Skipped, the build # job reports as skipped, which a required check counts as passed. changed: - runs-on: blacksmith-8vcpu-ubuntu-2404 + runs-on: ubuntu-latest timeout-minutes: 10 outputs: build: ${{ steps.touches.outputs.build }} @@ -72,7 +72,7 @@ jobs: build: needs: changed if: needs.changed.outputs.build == 'true' - runs-on: blacksmith-8vcpu-ubuntu-2404 + runs-on: ubuntu-latest # A cold build is tens of minutes and a warm one is a cache restore, so the spread is # wide and the ceiling has to sit above the cold case. What it is really for is the # other end: a `configure` that stops on a prompt would hold a runner for six hours @@ -90,22 +90,10 @@ jobs: go-version-file: go.mod cache: true - # cache-key is required as of v2 and names the sticky disk the builder keeps its state - # on, so it is one key per thing built, for the same reason the BuildKit scopes in - # Taskfile.yml are: share one and each build is handed the last one's disk and starts by - # discarding it. - # - # nofallback, because the default is to degrade. A builder that cannot be set up - # otherwise reports the error, falls back to a local one and carries on, so the run goes - # green having lost the disk — which is how v2's new required input arrived unnoticed. - # The cost of this is that an outage at a third party turns the lane red instead of - # making it slow, which is the trade this repository makes everywhere else: a build that - # did not do what it says is worth more as a failure than as a warning nobody reads. - - name: Setup Blacksmith Builder - uses: useblacksmith/setup-docker-builder@19215110ab936351210feebdfa5b440b4493e184 # v2.2.0 - with: - cache-key: spin-machine/qemu - nofallback: true + # A builder of its own (the docker-container driver): Docker's default one cannot export its + # cache. What it reads and writes is GitHub's cache, in the scopes Taskfile.yml decides + # (CACHE_BACKEND=gha), so the runner itself keeps nothing between runs. + - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 # What makes `--cache-to type=gha` do anything at all. # @@ -190,8 +178,8 @@ jobs: # The runtime image, tagged with the pinned QEMU version: what `task qemu:fetch` pulls, # so CI's argument check runs main's QEMU in seconds instead of compiling it. The # binaries are static, so the image around them carries nothing they need. It reads the - # builder stage the step above compiled from the `qemu` scope, and writes no cache: the - # runner's sticky disk already holds these layers. + # builder stage the step above compiled from the `qemu` scope, and writes no cache: that + # scope holds these layers already. - name: Publish the runtime image if: github.ref == 'refs/heads/main' && github.event_name != 'pull_request' run: go tool task qemu:publish diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6a48f93..102156a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -50,7 +50,7 @@ jobs: release: needs: compat if: github.ref == 'refs/heads/main' - runs-on: blacksmith-8vcpu-ubuntu-2404 + runs-on: ubuntu-latest # A warm run reads main's BuildKit cache and takes a few minutes (2m13s, 2026-09-30). The # ceiling is for the cold one, which compiles QEMU and the kernel, each tens of minutes. timeout-minutes: 180 @@ -74,19 +74,13 @@ jobs: go-version-file: go.mod cache: true - # Its own sticky disk, and not one of the lanes'. This job builds all four artefacts, so - # it would want every lane's disk at once and can only have one — and a release can - # overlap a lane, which sharing a key would make them contend for. The BuildKit scopes it - # reads are still the lanes', so what main already compiled is not compiled again; see - # the note in qemu.yml and CACHE_BACKEND in Taskfile.yml. - # And no silent fallback, which matters most here: a release that compiled QEMU and the - # kernel from cold because its builder quietly became a local one is hours, and it would - # look like nothing but a slow release. - - name: Setup Blacksmith Builder - uses: useblacksmith/setup-docker-builder@19215110ab936351210feebdfa5b440b4493e184 # v2.2.0 - with: - cache-key: spin-machine/release - nofallback: true + # What main's lanes already compiled is read, not compiled again: the BuildKit scopes are the + # lanes' (CACHE_BACKEND in Taskfile.yml); a cold release compiles QEMU and the kernel, each + # tens of minutes, which the ceiling above is for. + # A builder of its own (the docker-container driver): Docker's default one cannot export its + # cache. What it reads and writes is GitHub's cache, in the scopes Taskfile.yml decides + # (CACHE_BACKEND=gha), so the runner itself keeps nothing between runs. + - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 # Without this, `--cache-to type=gha` below is a silent no-op: the tokens BuildKit # needs reach JavaScript actions and not `run:` steps. See the long note in qemu.yml. diff --git a/.github/workflows/report.yml b/.github/workflows/report.yml index 3c7e59e..fe2d8b7 100644 --- a/.github/workflows/report.yml +++ b/.github/workflows/report.yml @@ -4,11 +4,12 @@ name: Report # through the feature matrix (boot/report_test.go), written as report.json, and compared with the # newest earlier release that has one. # -# A workflow of its own because it boots: a hosted runner has no /dev/kvm for it, and a time -# measured under emulation is not a time anybody's machine takes. The runner is one host, on -# purpose - boot times compare only between reports taken on the same kind of host, and compare -# says so when they were not. It needs /dev/kvm and /dev/vhost-vsock usable by its user, sudo -n -# for modprobe nbd and qemu-nbd, and gh (docs/releasing.md). +# A workflow of its own because it boots: GitHub's runners have no /dev/kvm for it, and a time +# measured under emulation is not a time anybody's machine takes. The runner is one instance type, +# on purpose - the bench fleet of spin-stack/ci, a c8i.4xlarge with nested virtualization - since +# boot times compare only between reports taken on the same kind of host, and compare says so +# when they were not. It needs /dev/kvm and /dev/vhost-vsock usable by its user, sudo -n for +# modprobe nbd and qemu-nbd, and gh (docs/releasing.md). # # release.yml calls it with publish, which uploads report.json to the release and appends the # comparison to its notes. Run by hand it measures any release - to check the runner, or to see @@ -36,7 +37,7 @@ permissions: jobs: report: - runs-on: [self-hosted, linux, x64, kvm] + runs-on: runs-on/fleet=bench/env=production timeout-minutes: 360 env: GH_TOKEN: ${{ github.token }} @@ -52,13 +53,17 @@ jobs: with: ref: ${{ inputs.ref || github.sha }} - # No Actions cache: this runner keeps Go's module and build caches on its own disk, and - # the cache service only stored another copy per branch, against the repository's limit. + # Go's caches come back from the fleet's cache (its S3 bucket): the instance starts with + # nothing. - name: Set up Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version-file: go.mod - cache: false + + # The guests' vsock transport, loaded and made this user's: a module on the fleet's kernel + # that nothing loads before the first QEMU opens /dev/vhost-vsock. + - name: Load vhost_vsock + run: go tool task vsock # What this runner can do, said before anything is measured: a row the report skips is one # this host lacks the device for, and the summary says which before the report does. diff --git a/AGENTS.md b/AGENTS.md index 8a8e26d..178f153 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -90,10 +90,10 @@ not silently become production behaviour. CPU affinity), document why it does not. - Remove or quarantine an experiment once it no longer has an active question. Keep the conclusion and durable evidence, not a permanent switch in the release path. -- A measurement that decides something is taken on the KVM runner (construct), through - `.github/workflows/lab.yml`, one run at a time: its three runners share one machine, and two - lab runs side by side measure each other. A laptop has neither the Spin OS kernel nor a quiet - host, so its numbers are leads, not findings. Compare with enough boots to beat the noise - +- A measurement that decides something is taken on the bench fleet (spin-stack/ci, one instance + type) through `.github/workflows/lab.yml`, one run at a time: two lab runs side by side measure + each other. A laptop is not a quiet host, so its numbers are leads, not findings, and a number + taken on another kind of host compares with nothing. Compare with enough boots to beat the noise - three a row moved a p50 by 5-11% between runs of one release (#81) - and alternate the variants rather than running them in blocks. - What the guest's memory and disk were measured to cost, and why each default is what it is, diff --git a/Taskfile.yml b/Taskfile.yml index 0c176fb..8c58273 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -92,13 +92,11 @@ vars: # are not in a plain `run:` step's environment, and without them `--cache-to type=gha` is # accepted, stores nothing, and says nothing. CACHE_BACKEND: '{{.CACHE_BACKEND | default "local"}}' - # Whether a build writes its cache back, and only CI's main needs it to. On Blacksmith the - # builder keeps BuildKit's own state on a sticky disk per cache-key (setup-docker-builder), - # so a lane re-reads what it built without any export; what the Actions cache adds is the - # one thing a disk cannot: the release, which builds everything with a builder of its own, - # reusing what main's lanes paid for. Exporting cost 88 s of "preparing build cache for - # export" on a kernel build (2026-09-30), on every PR and dispatch that warmed nothing a - # release reads. So the lanes export on a push to main, and nothing else does. + # Whether a build writes its cache back, and only CI's main needs it to: a pull request's + # build, a release and compat read main's scopes, and what they compiled nobody reads after. + # Exporting cost 88 s of "preparing build cache for export" on a kernel build (2026-09-30), + # on every PR and dispatch that warmed nothing a release reads. So the lanes export on a push + # to main, and nothing else does. CACHE_EXPORT: '{{.CACHE_EXPORT | default "true"}}' BUILDKIT_CACHE_DIR: '{{.ROOT_DIR}}/.cache/buildkit' @@ -308,10 +306,15 @@ tasks: Make /dev/vhost-vsock usable by this user, so verify:args can create the machine's vsock device. Without it, says so rather than failing: verify:args names the device unchecked. cmds: - # The device first: a kernel that builds it in - Spin OS's, which has no modules - has it - # from boot, and modprobe there has nothing to find. + # The module before the node is looked at: GitHub's runner has the static node before the + # module is loaded, the first open loads it, and udev then sets the node back to root's - + # QEMU got "Permission denied" after a chmod that saw the node and skipped the modprobe + # (2026-10-24). A kernel that builds it in - Spin OS's, which has no modules - has the + # device from boot, and modprobe there has nothing to find. - | - { [ -c /dev/vhost-vsock ] || sudo modprobe vhost_vsock; } && sudo chmod 0666 /dev/vhost-vsock || + sudo modprobe vhost_vsock 2>/dev/null || true + sudo udevadm settle 2>/dev/null || true + { [ -c /dev/vhost-vsock ] && sudo chmod 0666 /dev/vhost-vsock; } || echo "no vhost_vsock here; the check will name vhost-vsock-pci as unchecked" verify:args: diff --git a/boot/Taskfile.yml b/boot/Taskfile.yml index 378a249..e1921bb 100644 --- a/boot/Taskfile.yml +++ b/boot/Taskfile.yml @@ -43,7 +43,8 @@ tasks: gh release download "$tag" -R spin-stack/spin-machine -D "$dir" \ -p "$name.tar.gz" -p "$name.tar.gz.sha256" -p "$name.tar.gz.sigstore.json" (cd "$dir" && sha256sum -c "$name.tar.gz.sha256") - # cosign by its pin, built: the KVM runner this runs on has no envsubst for the action. + # cosign by its pin, built rather than installed by the action: one pin for a laptop + # and a runner alike. go run "github.com/sigstore/cosign/v3/cmd/cosign@$(go tool versions version cosign)" verify-blob \ --bundle "$dir/$name.tar.gz.sigstore.json" \ --certificate-identity https://github.com/spin-stack/spin-machine/.github/workflows/release.yml@refs/heads/main \ @@ -55,7 +56,7 @@ tasks: buildkit: desc: >- Use the BuildKit builder named compat, made the first time: a docker-container builder, - which keeps its cache in .cache/buildkit for the KVM runner's next run. + which reads and exports a cache (CACHE_BACKEND) where Docker's own cannot. cmds: - docker buildx inspect compat > /dev/null 2>&1 && docker buildx use compat || docker buildx create --use --name compat --driver docker-container diff --git a/docs/memory-and-disk.md b/docs/memory-and-disk.md index 82d4945..0e05719 100644 --- a/docs/memory-and-disk.md +++ b/docs/memory-and-disk.md @@ -169,16 +169,18 @@ B: blog), before measuring: ## Measuring it again -The probes are `go test` behind the `boot/` Task targets, run on construct by +The probes are `go test` behind the `boot/` Task targets, run on the bench fleet (spin-stack/ci) by `gh workflow run lab.yml -f experiment=boot:page-cache -f vars='REPS=3; SPIN_PAGE_CACHE_ONLY='` (caches, reclaim), optionally with `kernel_run`/`qemu_run` and `variant=beside|instead` for a build of a branch. The cold-boot, `memory.high`, reclaimers and first-write probes behind the other tables were deleted once answered; they are in `boot/pagecache_test.go`'s history. What the lab taught about itself: -- **Only construct's numbers count.** A laptop has neither the Spin OS kernel nor a quiet host. -- **One run at a time.** construct has three runners on one machine; two lab runs side by side - measure each other. Read results by run id, never by "the newest". +- **Only a quiet host's numbers count**, and only against numbers from the same kind of host: + the tables above are construct's, and the bench fleet's instance type is another host. A + laptop is neither. +- **One run at a time.** Two lab runs side by side measure each other; the bench fleet runs one. + Read results by run id, never by "the newest". - **A probe that mounts must unmount in `t.Cleanup`**: a loop mount left behind broke every later checkout on that runner. - **Unix socket paths are 108 bytes**: QMP sockets live under `/tmp`, not the lab's long diff --git a/docs/releasing.md b/docs/releasing.md index a8e3829..ae2de54 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -80,7 +80,7 @@ pinned Alpine, and the diff is the review. ## The feature matrix A release also says what it costs. After it publishes, the `report` job boots the tarball it -just published, on a self-hosted runner labelled `kvm`, through every combination of the +just published, on the bench fleet (spin-stack/ci: one instance type with `/dev/kvm`), through every combination of the machine's features under KVM (memory fixed or with a virtio-mem ceiling; vsock; a hotplug controller) and every boot variant of the image. It writes `report.json` beside the tarball: per row, the command line, the shape and fingerprint, whether QEMU ran it, and p50/p95 of each boot phase. It then