diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fd71b76d8..4d4a5f493 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -77,7 +77,7 @@ jobs: # queue forever — see the `commit-signatures` note below). Each output folds in # `.github/workflows/**`, so any change to CI itself triggers a full run. changes: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 # one LTS: path filter only, nothing here depends on the image timeout-minutes: 20 # paths-filter reads the PR's changed-file list from the API on # pull_request events; that needs pull-requests:read on top of the @@ -232,7 +232,7 @@ jobs: # advisory only, and deliberately NOT a required check. xfail-hint: name: Stale xfail hint (advisory) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 # one LTS: advisory text check, nothing here depends on the image timeout-minutes: 20 if: github.event_name == 'pull_request' steps: @@ -256,7 +256,15 @@ jobs: --commits-file /tmp/pr-commits.txt >> "$GITHUB_STEP_SUMMARY" build-and-test: - runs-on: ubuntu-latest + # Runs on both Ubuntu LTS images. The 24.04 leg keeps the bare name, + # which is what branch protection requires; the 26.04 leg reports as + # "build-and-test (ubuntu-26.04)". + name: ${{ matrix.os == 'ubuntu-24.04' && 'build-and-test' || format('build-and-test ({0})', matrix.os) }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, ubuntu-26.04] + runs-on: ${{ matrix.os }} timeout-minutes: 20 # sccache caches each rustc invocation (keyed by source + deps + flags) in an # Azure blob container, so unchanged workspace crates are restored instead of @@ -302,7 +310,10 @@ jobs: SCCACHE_AZURE_BLOB_CONTAINER: cache # Namespaced so the Linux and Windows lanes cannot collide, and so the # container stays legible if it is ever shared with another consumer. - SCCACHE_AZURE_KEY_PREFIX: rocm-cli/build-and-test + # Per image too: sccache keys on the compiler binary, which is the same + # rustup download on every image, so a proc macro linked against one + # Ubuntu release's glibc would otherwise be served to the other. + SCCACHE_AZURE_KEY_PREFIX: rocm-cli/build-and-test/${{ matrix.os }} SCCACHE_AZURE_RW_MODE: ${{ (github.event_name == 'push' || github.event_name == 'merge_group') && 'READ_WRITE' || 'READ_ONLY' }} # Don't spend per-OS build/test cycles unless the cheap lint gate is green. needs: [changes, clippy, prek] @@ -387,6 +398,10 @@ jobs: - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 if: needs.changes.outputs.heavy == 'true' + with: + # rust-cache keys on "Linux-x64" only; name the image so a target dir + # built against one Ubuntu release's glibc is never restored on another. + cache-key: ${{ matrix.os }} # Build the runtime binaries the smoke + acceptance steps below need (every # engine binary; smoke hard-fails on a missing one). Unit/integration tests @@ -479,8 +494,15 @@ jobs: # rust-cache lane (`cache-key` above) for the same reason. Revisit if this job # becomes a bottleneck. test: - name: Test (affected crates) - runs-on: ubuntu-latest + # Runs on both Ubuntu LTS images. The 24.04 leg keeps the bare name, + # which is what branch protection requires; the 26.04 leg reports as + # "Test (affected crates) (ubuntu-26.04)". + name: ${{ matrix.os == 'ubuntu-24.04' && 'Test (affected crates)' || format('Test (affected crates) ({0})', matrix.os) }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, ubuntu-26.04] + runs-on: ${{ matrix.os }} timeout-minutes: 20 # Same lint gate as the OS build jobs: don't spend a test cycle until the # cheap checks are green. @@ -506,7 +528,9 @@ jobs: # cache lanes. A subset build resolves Cargo features differently than # the whole workspace, so a shared cache would make the two evict and # recompile each other's artifacts on every alternation. - cache-key: ${{ github.event_name == 'pull_request' && 'affected-pr' || 'full' }} + # The runner image is part of the lane too, for the reason the other + # rust-cache keys in this workflow give. + cache-key: ${{ matrix.os }}-${{ github.event_name == 'pull_request' && 'affected-pr' || 'full' }} - name: Install cargo-nextest if: needs.changes.outputs.rust == 'true' @@ -864,7 +888,7 @@ jobs: # per-run artifacts for manual testing, not a publication. pr-linux-binaries: name: Build Linux PR binaries - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 # one LTS: the binaries are built in the manylinux container, not on the image # 30min: a cold containerised release build (image pull, yum install, # rustup install, then `cargo build --release` for 3 crates) plus none of # its steps below are individually bounded -- see the convention comment @@ -891,6 +915,10 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 + with: + # rust-cache keys on "Linux-x64" only; name the image so a target dir + # built against one Ubuntu release's glibc is never restored on another. + cache-key: ubuntu-24.04 - name: Cache Cargo home and build artifacts uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 @@ -970,7 +998,7 @@ jobs: # their own jobs (clippy, build-and-test, license-headers, docs-links, # powershell-lint, commit-signatures). name: prek (lint / hygiene) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 # one LTS: lint/hygiene only, nothing here depends on the image timeout-minutes: 20 # Always-on: prek is the cheapest gate (no compile) and its hygiene hooks run # on every file type, so it is relevant to any change — including docs-only @@ -1013,8 +1041,15 @@ jobs: # # No GPU, no ROCm, no tailnet: the remote's `rocm` and `tailscale` are # stand-ins that answer in the shapes the real tools do. - name: remote control channel (containerised) - runs-on: ubuntu-latest + # Runs on both Ubuntu LTS images. The 24.04 leg keeps the bare name, + # which is what branch protection requires; the 26.04 leg reports as + # "remote control channel (containerised) (ubuntu-26.04)". + name: ${{ matrix.os == 'ubuntu-24.04' && 'remote control channel (containerised)' || format('remote control channel (containerised) ({0})', matrix.os) }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, ubuntu-26.04] + runs-on: ${{ matrix.os }} timeout-minutes: 30 # Gated like every other heavy job here: a fast lint failure should # short-circuit before this spends 30 minutes on a container and a cold @@ -1024,8 +1059,8 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - # Docker and the OpenSSH client are both present on ubuntu-latest; jq is - # the only extra, and only for asserting on JSON contracts. + # Docker and the OpenSSH client are both present on the hosted Ubuntu + # images; jq is the only extra, and only for asserting on JSON contracts. - name: Install jq run: sudo apt-get update && sudo apt-get install -y jq @@ -1037,6 +1072,10 @@ jobs: # whole workspace cold against a different compiler from the rest of CI, # inside a 30-minute timeout. - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 + with: + # rust-cache keys on "Linux-x64" only; name the image so a target dir + # built against one Ubuntu release's glibc is never restored on another. + cache-key: ${{ matrix.os }} # The same container, driven by the real binary this time: discover a # target, probe it, serve, publish, reconcile status, re-publish after an @@ -1051,7 +1090,7 @@ jobs: run: tests/remote-ssh/run-e2e.sh clippy: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 # one LTS: compile-only lint with the pinned toolchain; the image does not change its result timeout-minutes: 20 # sccache caches each rustc invocation across runs (see build-and-test for the # choice of backend and the probe). clippy compiles the whole workspace with @@ -1065,12 +1104,13 @@ jobs: # build job's, and workspace crates go through clippy-driver, whose hash # differs from rustc's. Crates that must be fully linked for both (proc # macros, and anything a build script needs) do hit. Sharing cannot poison - # either job: the compiler binary is part of every key. + # either job: the compiler binary is part of every key. It is the 24.04 + # leg's prefix specifically, since this job runs on that image. env: RUSTC_WRAPPER: sccache SCCACHE_AZURE_CONNECTION_STRING: ${{ secrets.CACHE_AZURE_CONNECTION_STRING }} SCCACHE_AZURE_BLOB_CONTAINER: cache - SCCACHE_AZURE_KEY_PREFIX: rocm-cli/build-and-test + SCCACHE_AZURE_KEY_PREFIX: rocm-cli/build-and-test/ubuntu-24.04 SCCACHE_AZURE_RW_MODE: ${{ (github.event_name == 'push' || github.event_name == 'merge_group') && 'READ_WRITE' || 'READ_ONLY' }} needs: changes # A manual E2E dispatch only exercises the E2E jobs; skip the rest. @@ -1125,6 +1165,10 @@ jobs: - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 if: needs.changes.outputs.rust == 'true' + with: + # rust-cache keys on "Linux-x64" only; name the image so a target dir + # built against one Ubuntu release's glibc is never restored on another. + cache-key: ubuntu-24.04 - name: Clippy if: needs.changes.outputs.rust == 'true' @@ -1161,7 +1205,7 @@ jobs: third-party-notices: name: Third-party notices current - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 # one LTS: generated-manifest check, nothing here depends on the image timeout-minutes: 20 needs: changes # A manual E2E dispatch only exercises the E2E jobs; skip the rest. @@ -1171,6 +1215,10 @@ jobs: - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 if: needs.changes.outputs.tpn == 'true' + with: + # rust-cache keys on "Linux-x64" only; name the image so a target dir + # built against one Ubuntu release's glibc is never restored on another. + cache-key: ubuntu-24.04 # Pin the cargo-about version so the check compares against the exact same # generator contributors use; a different version can format the notices @@ -1184,7 +1232,9 @@ jobs: uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ~/.cargo/bin/cargo-about - key: ${{ runner.os }}-cargo-about-0.9.1 + # The image is in the key: this caches a binary `cargo install` linked + # against the image's glibc. + key: ${{ runner.os }}-ubuntu-24.04-cargo-about-0.9.1 - name: Install cargo-about if: needs.changes.outputs.tpn == 'true' && steps.cache-cargo-about.outputs.cache-hit != 'true' @@ -1202,7 +1252,7 @@ jobs: # request waiting on a check that can no longer report. Rename and update # protection together, as their own change. name: Coverage (rocm-dash crates, ratcheted) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 # one LTS: one measurement against the committed floors; Test (affected crates) runs the tests on both LTSes # Raised from 20: the gate now instruments the whole workspace rather than # four crates, so the build it waits on is several times larger. timeout-minutes: 45 @@ -1235,9 +1285,11 @@ jobs: ~/.cargo/registry ~/.cargo/git target - key: ${{ runner.os }}-cargo-cov-${{ hashFiles('**/Cargo.lock') }} + # The image is in the key: target/ holds build scripts and proc macros + # linked against the image's glibc. + key: ${{ runner.os }}-ubuntu-24.04-cargo-cov-${{ hashFiles('**/Cargo.lock') }} restore-keys: | - ${{ runner.os }}-cargo-cov- + ${{ runner.os }}-ubuntu-24.04-cargo-cov- # Per-crate floors over every workspace crate, pinned in # `coverage-floors.toml` and enforced by the xtask. The crate list lives @@ -1317,7 +1369,7 @@ jobs: # prompt. skill-evals: name: Skill checks (skillscope) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 # one LTS: lint/hygiene only, nothing here depends on the image timeout-minutes: 10 needs: changes # A manual E2E dispatch only exercises the E2E jobs; skip the rest. @@ -1374,7 +1426,7 @@ jobs: license-headers: name: License header check (hawkeye) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 # one LTS: lint/hygiene only, nothing here depends on the image timeout-minutes: 20 env: # Install a pinned prebuilt hawkeye binary instead of pulling the @@ -1418,7 +1470,7 @@ jobs: docs-links: name: Markdown links resolve (lychee) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 # one LTS: lint/hygiene only, nothing here depends on the image timeout-minutes: 20 # Deliberately NOT gated by the `changes` path filter: a markdown link can # go stale when the file or heading it points at moves, without the @@ -1440,7 +1492,7 @@ jobs: docs-build: name: Sphinx docs build (-W) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 # one LTS: docs build only, nothing here depends on the image timeout-minutes: 20 needs: [changes] # A manual E2E dispatch only exercises the E2E jobs; skip the rest. Off a @@ -1471,7 +1523,7 @@ jobs: commit-signatures: name: Commit signatures + sign-off - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 # one LTS: commit metadata check, nothing here depends on the image timeout-minutes: 20 # Runs on pull requests and in the merge queue. Both define a base that # bounds the commit range. Handling `merge_group` matters once this is a @@ -1493,6 +1545,10 @@ jobs: ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.event.merge_group.head_sha }} - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 + with: + # rust-cache keys on "Linux-x64" only; name the image so a target dir + # built against one Ubuntu release's glibc is never restored on another. + cache-key: ubuntu-24.04 - name: Verify commits are signed and signed-off # `--require-verified` shells out to the `gh` CLI; pass the job token so @@ -1527,9 +1583,16 @@ jobs: # scenarios resolve to skip here; known bugs resolve to xfail from # expectations.toml. e2e: - name: E2E tests timeout-minutes: 15 - runs-on: ubuntu-latest + # Runs on both Ubuntu LTS images. The 24.04 leg keeps the bare name, + # which is what branch protection requires; the 26.04 leg reports as + # "E2E tests (ubuntu-26.04)". + name: ${{ matrix.os == 'ubuntu-24.04' && 'E2E tests' || format('E2E tests ({0})', matrix.os) }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, ubuntu-26.04] + runs-on: ${{ matrix.os }} needs: [changes, build-and-test] # This is a REQUIRED check, so — like every sibling required job — the job # itself must always run (except on a manual dispatch that didn't select the @@ -1546,15 +1609,21 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 # Whether the real work runs this time. On push/PR/merge_group: only when - # build-and-test succeeded and the change is heavy. On workflow_dispatch: - # build-and-test is skipped, so tolerate that and rely on the job-level - # platform gate above. + # build-and-test ran to a verdict and the change is heavy. On + # workflow_dispatch: build-and-test is skipped, so tolerate that and rely + # on the job-level platform gate above. + # + # "Ran to a verdict", not "succeeded": build-and-test is a matrix over + # both images and its result is the aggregate, so a failure on one image + # would otherwise skip this job's work on the other too — and the required + # 24.04 leg would report green without having run a scenario. - name: Decide whether to run id: gate run: | if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then echo "run=true" >> "$GITHUB_OUTPUT" - elif [ "${{ needs.build-and-test.result }}" = "success" ] \ + elif { [ "${{ needs.build-and-test.result }}" = "success" ] \ + || [ "${{ needs.build-and-test.result }}" = "failure" ]; } \ && [ "${{ needs.changes.outputs.heavy }}" = "true" ]; then echo "run=true" >> "$GITHUB_OUTPUT" else @@ -1567,6 +1636,10 @@ jobs: - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 if: steps.gate.outputs.run == 'true' + with: + # rust-cache keys on "Linux-x64" only; name the image so a target dir + # built against one Ubuntu release's glibc is never restored on another. + cache-key: ${{ matrix.os }} # The workspace test job selects only affected crates and runs the cucumber # harness, not the library's own unit tests — run the capability/expectation @@ -1587,13 +1660,28 @@ jobs: E2E_INCLUDE_DOCKER: "1" run: cargo xtask e2e + # One upload step per image rather than one templated `name:`, as in + # e2e-selfhosted.yml: the artifact-name contract in `xtask::e2e_report` + # reads `name: e2e-*` lines literally. Artifact names are unique per run. + # The 24.04 leg keeps `e2e-report`, which the consolidated report below + # picks up through its `*-report` glob. The 26.04 leg's artifact is named + # outside both that glob and the `e2e-` namespace: the report has one + # mock platform, and two inputs labelled as it would collide. It is + # uploaded for inspection only. - name: Upload E2E report - if: always() && steps.gate.outputs.run == 'true' + if: always() && steps.gate.outputs.run == 'true' && matrix.os == 'ubuntu-24.04' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: e2e-report path: tests/e2e-cucumber/results/ + - name: Upload E2E results (ubuntu-26.04) + if: always() && steps.gate.outputs.run == 'true' && matrix.os == 'ubuntu-26.04' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ubuntu-26.04-e2e-results + path: tests/e2e-cucumber/results/ + # Consolidate the mock (GitHub-hosted) platform's report into the cross-platform # report: a platform × tier matrix in the run Summary plus a single merged HTML # artifact. `if: always()` so a failing platform still appears; runs on @@ -1605,7 +1693,7 @@ jobs: # glob still auto-includes any new GitHub-hosted platform added here. e2e-report: name: E2E consolidated report - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 # one LTS: only merges report.json files # Bound the job (like every E2E job) so a stalled artifact download can't hang # a hosted runner indefinitely. timeout-minutes: 15 @@ -1622,6 +1710,10 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 + with: + # rust-cache keys on "Linux-x64" only; name the image so a target dir + # built against one Ubuntu release's glibc is never restored on another. + cache-key: ubuntu-24.04 # Pull every e2e artifact matching the glob. `xtask e2e-report` turns each # into one labeled platform and the `*-report` glob makes new platforms diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 91edef643..0b302dcaf 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -17,7 +17,7 @@ permissions: jobs: analyze: name: Analyze (${{ matrix.language }}) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 # one LTS: static analysis; the image does not change what it finds permissions: security-events: write packages: read diff --git a/.github/workflows/demo-gifs.yml b/.github/workflows/demo-gifs.yml index fb97da4bc..537dd6639 100644 --- a/.github/workflows/demo-gifs.yml +++ b/.github/workflows/demo-gifs.yml @@ -19,11 +19,15 @@ concurrency: jobs: render: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 # one LTS: renders documentation GIFs steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 + with: + # rust-cache keys on "Linux-x64" only; name the image so a target dir + # built against one Ubuntu release's glibc is never restored on another. + cache-key: ubuntu-24.04 # VHS + its runtime deps. ttyd/ffmpeg from apt; vhs pinned via `go install` # (Go is preinstalled on the runner). Chrome (used by VHS to rasterize diff --git a/.github/workflows/dependabot-manifests-commit.yml b/.github/workflows/dependabot-manifests-commit.yml index f0f942f00..2811dbab6 100644 --- a/.github/workflows/dependabot-manifests-commit.yml +++ b/.github/workflows/dependabot-manifests-commit.yml @@ -26,7 +26,7 @@ jobs: github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.actor.login == 'dependabot[bot]' && github.event.workflow_run.event == 'pull_request' - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 # one LTS: commits generated files permissions: actions: read contents: write diff --git a/.github/workflows/dependabot-manifests.yml b/.github/workflows/dependabot-manifests.yml index 5a8b40588..1140f34e8 100644 --- a/.github/workflows/dependabot-manifests.yml +++ b/.github/workflows/dependabot-manifests.yml @@ -50,7 +50,7 @@ jobs: generate: name: Regenerate manifests if: github.actor == 'dependabot[bot]' - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 # one LTS: regenerates manifests, nothing here depends on the image permissions: contents: read steps: @@ -61,6 +61,10 @@ jobs: ref: ${{ github.event.pull_request.head.sha }} - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 + with: + # rust-cache keys on "Linux-x64" only; name the image so a target dir + # built against one Ubuntu release's glibc is never restored on another. + cache-key: ubuntu-24.04 # Same version and cache key as ci.yml's third-party-notices job: the # notices must be byte-identical to what that job's `--check` regenerates, @@ -70,7 +74,7 @@ jobs: uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ~/.cargo/bin/cargo-about - key: ${{ runner.os }}-cargo-about-0.9.1 + key: ${{ runner.os }}-ubuntu-24.04-cargo-about-0.9.1 - name: Install cargo-about if: steps.cache-cargo-about.outputs.cache-hit != 'true' diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index 8cdb2b520..cc18c5e63 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -86,7 +86,7 @@ jobs: # GPU lanes on a doc-only PR. Off pull_request (push/merge_group) heavy is # forced true, matching ci.yml. changes: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 # one LTS: path filter only, nothing here depends on the image permissions: contents: read pull-requests: read @@ -1631,7 +1631,7 @@ jobs: # so a failing/skipped self-hosted platform still appears in the grid. e2e-report: name: E2E consolidated report (self-hosted) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 # one LTS: only merges report.json files timeout-minutes: 15 # `actions: read` lets the missing-report check list this run's artifacts. permissions: @@ -1656,6 +1656,10 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 + with: + # rust-cache keys on "Linux-x64" only; name the image so a target dir + # built against one Ubuntu release's glibc is never restored on another. + cache-key: ubuntu-24.04 # Pull every self-hosted e2e artifact matching the glob; `xtask e2e-report` # turns each into one labeled platform and the `*-report` glob picks up any diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 10e179c43..5a9fc5893 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -32,7 +32,7 @@ env: jobs: nightly: name: Build Linux nightly assets - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 # one LTS: the assets are built in the manylinux container, not on the image outputs: skip: ${{ steps.check.outputs.skip }} date: ${{ steps.meta.outputs.date }} @@ -47,6 +47,10 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 + with: + # rust-cache keys on "Linux-x64" only; name the image so a target dir + # built against one Ubuntu release's glibc is never restored on another. + cache-key: ubuntu-24.04 - name: Verify pinned keys match the signing key run: cargo xtask verify-pinned-keys @@ -274,7 +278,7 @@ jobs: publish-nightly: name: Publish nightly - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 # one LTS: publishing only needs: - nightly - windows-nightly @@ -343,7 +347,7 @@ jobs: cleanup-staging-nightly: name: Clean failed nightly staging release - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 # one LTS: release cleanup only needs: - nightly - windows-nightly @@ -1513,7 +1517,7 @@ jobs: # already does (e2e-selfhosted.yml), against the nightly artifacts. e2e-report-nightly: name: E2E consolidated report (nightly) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 # one LTS: only merges report.json files timeout-minutes: 15 # Narrower than the workflow-level `contents: write`, which exists for the # release-publishing jobs. This one checks out, downloads artifacts, runs an @@ -1539,6 +1543,10 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 + with: + # rust-cache keys on "Linux-x64" only; name the image so a target dir + # built against one Ubuntu release's glibc is never restored on another. + cache-key: ubuntu-24.04 # `*-report` matches the artifacts above, and will pick up any # nightly platform added later without a change here — provided it is @@ -1627,7 +1635,7 @@ jobs: # branch cannot overwrite the page, and never from an empty or cancelled run. publish-e2e-wiki: name: Publish E2E support matrix to wiki - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 # one LTS: publishing only timeout-minutes: 10 permissions: contents: write diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 139e4fc4c..52420d824 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -36,13 +36,17 @@ env: jobs: release: name: Build Linux release assets - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 # one LTS: the assets are built in the manylinux container, not on the image outputs: version: ${{ steps.version.outputs.value }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 + with: + # rust-cache keys on "Linux-x64" only; name the image so a target dir + # built against one Ubuntu release's glibc is never restored on another. + cache-key: ubuntu-24.04 - name: Verify pinned keys match the signing key run: cargo xtask verify-pinned-keys @@ -263,7 +267,7 @@ jobs: publish-release: name: Publish release - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 # one LTS: publishing only needs: - release - windows-release diff --git a/docs/ci-hardware-testing.md b/docs/ci-hardware-testing.md index 93384d6f8..4f3ff762e 100644 --- a/docs/ci-hardware-testing.md +++ b/docs/ci-hardware-testing.md @@ -6,7 +6,7 @@ SPDX-License-Identifier: MIT # CI hardware (GPU / WSL) testing -The hosted CI (`ubuntu-latest`, `windows-latest`) builds and unit-tests every +The hosted CI (`ubuntu-24.04` and `ubuntu-26.04`, `windows-latest`) builds and unit-tests every shipping target natively, but GitHub-hosted runners have no AMD GPU. A dedicated hardware layer covers that gap by running the same cucumber-rs E2E suite on dedicated self-hosted runners with real AMD GPUs. @@ -61,7 +61,7 @@ separate tier flag or tag filter to maintain. | Job | Workflow | Platform | Runner labels | |---|---|---|---| -| `e2e` | `ci.yml` | Mock (no GPU) | GitHub-hosted `ubuntu-latest` | +| `e2e` | `ci.yml` | Mock (no GPU) | GitHub-hosted `ubuntu-24.04` and `ubuntu-26.04` | | `e2e-gpu` | `e2e-selfhosted.yml` | MI300X (AMD Instinct, bare-metal Linux) | self-hosted `[self-hosted, linux, mi300x]` | | `e2e-gpu-strix-ubuntu` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on Ubuntu | self-hosted `[self-hosted, linux, devlab-dispatch, strix-halo]` | | `e2e-gpu-strix-windows` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on Windows 11 | self-hosted `[self-hosted, windows, devlab-dispatch, strix-halo]` | diff --git a/scripts/reclaim-gpu.sh b/scripts/reclaim-gpu.sh index 09a89d2e2..90ca4b192 100755 --- a/scripts/reclaim-gpu.sh +++ b/scripts/reclaim-gpu.sh @@ -453,23 +453,40 @@ assert_clear() { # Spawn a decoy whose argv[0] is a real-shaped path, so the match is tested # against the same string a live engine process would present. # -# The executable is a copy of bash, not of sleep: where coreutils is a multicall -# binary (uutils, the default since Ubuntu 25.10), /bin/sleep picks the tool to -# run from argv[0], so a copy named llama-server prints "unknown program" and -# exits before the self-test can see it. bash ignores the name it runs under. -# The body is a loop, not `sleep 300`, because bash execs a lone simple command -# in place and argv[0] would become `sleep`. Its `sleep 1` children name no E2E -# root, so they are never selected, and a TERM still ends the decoy. +# The bash running this script, started under that argv[0] with `exec -a`, so +# no binary is copied and nothing depends on how coreutils is built. Not sleep: +# where coreutils is a multicall binary (uutils, the default since Ubuntu 25.10), +# /bin/sleep picks the tool to run from the name it was started under, so a copy +# named llama-server prints "unknown program" and exits before the self-test can +# see it, and uutils reads that name from argv[0] on some targets and from the +# executed path on others. bash ignores its argv[0] unless it is `sh`. +# +# The body is the `read` builtin on a FIFO the shell holds open for reading AND +# writing, so the read never sees end of file. Being a builtin, it forks +# nothing: the decoy is this one pid, a TERM ends it like the engine it stands +# in for, and killing it leaves no child behind. spawn_decoy() { local path="$1" mkdir -p "$(dirname "${path}")" - cp "$(command -v bash)" "${path}" - # Detach stdio: this runs inside a command substitution, and a background - # child holding the capture pipe open would block the caller until it exits. - "${path}" -c 'for _ in {1..300}; do sleep 1; done' >/dev/null 2>&1 & + mkfifo "${path}.hold" + # Detach stdout/stderr: this runs inside a command substitution, and a + # background child holding the capture pipe open would block the caller until + # it exits. + (exec -a "${path}" "${BASH}" -c 'read -r _' <>"${path}.hold" >/dev/null 2>&1) & echo $! } +# Fail, naming the decoy, when one is no longer running. A decoy that died at +# spawn otherwise surfaces later as a selection or containment FAIL that blames +# the rules — which is how ubuntu-26.04's multicall `sleep` first showed up. +selftest_require_alive() { + local pid="$1" + local what="$2" + process_alive "${pid}" && return 0 + echo "FAIL: ${what} decoy (pid=${pid}) is not running; it exited at spawn, so the checks that read it test nothing" + return 1 +} + # Spawn a process that becomes a ZOMBIE and stays one, plus the keeper holding # it in that state. Echoes " "; the zombie pid is the # literal string `none` if one could not be produced — NOT an empty field, for @@ -748,8 +765,9 @@ self_test() { # here — including one a later edit adds, which is the case this guards. # # The flip side is worth knowing: because a broken spawn does not abort, it - # yields a live-looking pid for a process that died immediately, and surfaces - # later as a confusing assertion failure rather than as "the spawn failed". + # yields a live-looking pid for a process that died immediately. The + # selftest_require_alive calls after the spawns turn that into a FAIL naming + # the decoy, rather than a confusing assertion failure further down. prewarm_pid="$(spawn_decoy "${prewarm_decoy}")" selftest_track_decoy "${prewarm_pid}" workload_pid="$(spawn_decoy "${workload_decoy}")" @@ -773,8 +791,9 @@ self_test() { # `selftest_track_decoy` call is silent AND leaks for real: both the EXIT trap # and the final kill read this same list, so an untracked decoy is signalled # by nothing. It survives even a GREEN run — measured, the `/workload` fixture - # outlives the suite by its full 300s, because `reclaim` is supposed to spare - # that one and the tracked list is the only other thing that would kill it. + # outlived the suite (spawn_decoy's decoys never exit on their own), because + # `reclaim` is supposed to spare that one and the tracked list is the only + # other thing that would kill it. # (An earlier version of this comment claimed such a decoy "is still killed at # the end". It is not, and it is only incidentally true for the three fixtures # `reclaim` itself signals.) @@ -792,6 +811,12 @@ self_test() { fi # Give the decoys a moment to appear in /proc with their full argv. sleep 1 + selftest_require_alive "${prewarm_pid}" 'pre-warm' || failures=$((failures + 1)) + selftest_require_alive "${workload_pid}" '/workload' || failures=$((failures + 1)) + selftest_require_alive "${harness_pid}" 'harness' || failures=$((failures + 1)) + selftest_require_alive "${stubborn_pid}" 'stubborn' || failures=$((failures + 1)) + selftest_require_alive "${outside_pid}" 'bystander' || failures=$((failures + 1)) + selftest_require_alive "${delimiter_pid}" 'delimiter' || failures=$((failures + 1)) # 1. Regression guard: the patterns this script replaced could not see a # pre-warm engine process. If this ever matches, the decoy stopped being @@ -1301,6 +1326,7 @@ self_test() { # carries the full argv, and a read that lands in between sees a command line # that matches nothing — which would report this check as a pass. sleep 1 + selftest_require_alive "${late_pid}" 'late leak' || failures=$((failures + 1)) assert_clear_rc=0 assert_clear_out="$(assert_clear 2>&1)" || assert_clear_rc=$? if [[ "${assert_clear_rc}" -ne 0 ]]; then @@ -1330,7 +1356,7 @@ self_test() { # Includes prewarm_pid: on a GREEN run reclaim has already killed it, but on a # FAILED run it was not selected, and it would otherwise outlive the scratch - # tree for its full 300s as an orphan. Killing the zombie's keeper lets init + # tree as an orphan that never exits on its own. Killing the zombie's keeper lets init # reap the zombie itself. kill -KILL "${decoy_pids[@]}" 2>/dev/null || true diff --git a/xtask/src/env_mutation_contract.rs b/xtask/src/env_mutation_contract.rs index d035f8042..1f24c38b3 100644 --- a/xtask/src/env_mutation_contract.rs +++ b/xtask/src/env_mutation_contract.rs @@ -20,7 +20,7 @@ //! //! "the Linux lanes use nextest" would be too strong: `coverage` //! (`cargo llvm-cov`, no `--nextest`) and `e2e` (`cargo test -p e2e-cucumber -//! --lib`) are threaded single-process harnesses on `ubuntu-latest` too. They +//! --lib`) are threaded single-process harnesses on the hosted Ubuntu runners too. They //! happen not to reach the crates that raced, which is luck rather than //! design — so this guard covers the whole tree rather than one crate. //! diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index 4962284e7..66e2110ca 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -751,6 +751,232 @@ trigger-a-workflow#triggering-a-workflow-from-a-workflow" } } + /// Every workflow file as `(file name, text)`, read from the directory + /// rather than from a list, so a workflow added later is covered too. + fn all_workflows() -> Vec<(String, String)> { + let dir = repo_root().join(".github/workflows"); + let mut names: Vec = std::fs::read_dir(&dir) + .unwrap_or_else(|e| panic!("reading {}: {e}", dir.display())) + .map(|entry| entry.expect("workflow dir entry").file_name()) + .map(|name| name.to_string_lossy().into_owned()) + .filter(|name| { + Path::new(name).extension().is_some_and(|ext| { + ext.eq_ignore_ascii_case("yml") || ext.eq_ignore_ascii_case("yaml") + }) + }) + .collect(); + names.sort(); + names + .into_iter() + .map(|name| { + let text = read_workflow(&name); + (name, text) + }) + .collect() + } + + /// The ids of every job in a workflow's `jobs:` block, in file order. + fn workflow_job_ids(text: &str) -> Vec { + top_level_block(text, "jobs") + .lines() + .filter(|line| indent_of(line) == 2 && !line.trim_start().starts_with('#')) + .filter_map(|line| line.trim().strip_suffix(':')) + .map(str::to_owned) + .collect() + } + + /// `(line number, line)` for every line naming `ubuntu-latest` outside a + /// comment, wherever it sits: a `runs-on`, a matrix list, an `include`. + fn moving_ubuntu_label_lines(text: &str) -> Vec<(usize, String)> { + text.lines() + .enumerate() + .filter(|(_, line)| !line.trim_start().starts_with('#')) + .filter(|(_, line)| strip_comment(line).contains("ubuntu-latest")) + .map(|(i, line)| (i + 1, line.trim().to_owned())) + .collect() + } + + /// No hosted job may run on `ubuntu-latest`. + /// + /// The label moves. When GitHub repointed it from ubuntu-24.04 to + /// ubuntu-26.04, every merge-group run went red overnight on a change + /// nobody in this repository made: 26.04's uutils coreutils broke the + /// reclaim decoys. Linux jobs name their images instead — the test and build + /// jobs run on both LTS releases, the rest on one — so a new release is + /// adopted by an edit to this repository, which CI then exercises. + #[test] + fn no_workflow_runs_on_the_moving_ubuntu_latest_label() { + let workflows = all_workflows(); + assert!( + workflows.iter().any(|(name, _)| name == "ci.yml"), + "expected ci.yml among the workflows (directory scan sanity check)" + ); + for (name, text) in workflows { + let hits = moving_ubuntu_label_lines(&text); + assert!( + hits.is_empty(), + "{name} names `ubuntu-latest`, which GitHub repoints to each new Ubuntu \ + release without notice; name `ubuntu-24.04` and/or `ubuntu-26.04` instead \ + (lines: {hits:?})" + ); + } + } + + #[test] + fn moving_ubuntu_label_detector_flags_values_but_not_comments() { + let yaml = "\ +jobs: + # ubuntu-latest moved under us once; see the contract test. + a: + runs-on: ubuntu-24.04 # not ubuntu-latest + b: + runs-on: ubuntu-latest + c: + strategy: + matrix: + os: [ubuntu-24.04, ubuntu-latest] +"; + let hits = moving_ubuntu_label_lines(yaml); + assert_eq!( + hits, + vec![ + (6, "runs-on: ubuntu-latest".to_owned()), + (10, "os: [ubuntu-24.04, ubuntu-latest]".to_owned()), + ] + ); + } + + /// The `ci.yml` jobs that run on both Ubuntu LTS images, with the name each + /// reported before the split. Branch protection requires these names + /// verbatim, so the 24.04 leg must keep reporting them. + const LTS_SPLIT_JOBS: [(&str, &str); 4] = [ + ("build-and-test", "build-and-test"), + ("test", "Test (affected crates)"), + ("e2e", "E2E tests"), + ("remote-ssh", "remote control channel (containerised)"), + ]; + + /// A split job keeps its required check, and also runs on 26.04. + /// + /// A plain matrix renames every leg to ` ()`, so the bare name + /// branch protection waits for would never report and every pull request + /// would sit on a missing required check. The 24.04 leg therefore keeps the + /// bare name and only the 26.04 leg gets a suffix. + #[test] + fn lts_split_jobs_keep_their_required_name_on_24_04() { + let ci = read_workflow("ci.yml"); + for (job, context) in LTS_SPLIT_JOBS { + let block = job_block(&ci, job); + let expected = format!( + "name: ${{{{ matrix.os == 'ubuntu-24.04' && '{context}' || \ + format('{context} ({{0}})', matrix.os) }}}}" + ); + assert!( + block.lines().any(|line| line.trim() == expected), + "ci.yml job `{job}` must report as `{context}` on ubuntu-24.04 — the name \ + branch protection requires — and with an image suffix on ubuntu-26.04; \ + expected the line `{expected}`" + ); + assert_eq!( + flattened_values(block, "runs-on"), + vec!["${{ matrix.os }}".to_owned()], + "ci.yml job `{job}` must run on its matrix image" + ); + let os = flattened_values(block, "os"); + assert_eq!( + os.len(), + 1, + "ci.yml job `{job}` declares one `os` matrix axis" + ); + assert_eq!( + flattened_list_items(&os[0]), + vec!["ubuntu-24.04".to_owned(), "ubuntu-26.04".to_owned()], + "ci.yml job `{job}` must run on both Ubuntu LTS images" + ); + assert!( + block.lines().any(|line| line.trim() == "fail-fast: false"), + "ci.yml job `{job}`: one image failing must not cancel the other's leg" + ); + } + } + + /// The runner image a hosted Linux job's caches must be keyed by, as it + /// appears in a key: the literal image, or `matrix.os` for a split job. + /// `None` for a job that is not a hosted Linux job. + fn hosted_linux_image_token(job_block_text: &str) -> Option { + let runs_on = flattened_values(job_block_text, "runs-on"); + let value = runs_on.first()?; + if value == "${{ matrix.os }}" { + Some("matrix.os".to_owned()) + } else if value.starts_with("ubuntu-") { + Some(value.clone()) + } else { + None + } + } + + /// Every compiled-artifact cache on a hosted Linux job is keyed by the + /// runner image. + /// + /// rust-cache keys on the OS family and architecture only ("Linux-x64"), + /// and sccache on the compiler binary, which is the same rustup download on + /// every image. Without the image in the key, a target dir or a proc macro + /// linked against 26.04's glibc is restored on 24.04, where it cannot load. + /// Covers `setup-rust-toolchain`'s `cache-key` (unless caching is off), the + /// sccache key prefix, and `actions/cache` keys over cargo output; the + /// manylinux release caches are exempt, as those builds run in a container. + #[test] + fn hosted_linux_compile_caches_are_keyed_by_runner_image() { + let mut checked = 0; + for (name, text) in all_workflows() { + for job in workflow_job_ids(&text) { + let block = job_block(&text, &job); + let Some(image) = hosted_linux_image_token(block) else { + continue; + }; + for (at, _) in block.match_indices("actions-rust-lang/setup-rust-toolchain@") { + let step = &block[at..]; + let step = step.find("\n - ").map_or(step, |end| &step[..end]); + if step.lines().any(|line| line.trim() == "cache: false") { + continue; + } + checked += 1; + let keys = flattened_values(step, "cache-key"); + assert!( + keys.iter().any(|key| key.contains(&image)), + "{name} job `{job}`: setup-rust-toolchain's rust-cache must carry \ + `cache-key` naming the runner image ({image}); its own key is only \ + \"Linux-x64\", so a target dir built on another Ubuntu release \ + would be restored here. Found: {keys:?}" + ); + } + for prefix in flattened_values(block, "SCCACHE_AZURE_KEY_PREFIX") { + checked += 1; + assert!( + prefix.contains(&image), + "{name} job `{job}`: SCCACHE_AZURE_KEY_PREFIX `{prefix}` must name \ + the runner image ({image})" + ); + } + for key in flattened_values(block, "key") { + if !key.contains("cargo") || key.contains("manylinux") { + continue; + } + checked += 1; + assert!( + key.contains(&image), + "{name} job `{job}`: cache key `{key}` holds cargo output and must \ + name the runner image ({image})" + ); + } + } + } + assert!( + checked > 0, + "no hosted Linux compile cache found (extractor sanity check)" + ); + } + #[test] fn self_hosted_workflow_owns_the_gpu_lanes() { let sh = read_workflow("e2e-selfhosted.yml"); @@ -2842,6 +3068,76 @@ fi Path::new("/proc/self/cmdline").exists() } + /// Starts a stand-in engine process whose command line begins with + /// `argv0`, and returns it only once it is seen alive under that name. + /// + /// The rule `--assert-clear` applies reads `/proc//cmdline`, so the + /// decoy only has to present the path there. It is `/bin/sh` run under that + /// argv[0], parked in the `read` builtin on a pipe this test holds. + /// + /// Nothing is copied, so nothing depends on how coreutils is built. A copy + /// of `/bin/sleep` named `llama-server` exits at once on ubuntu-26.04, whose + /// `/bin/sleep` is the uutils multicall binary (`coreutils: unknown program + /// 'llama-server'`); and `CommandExt::arg0` on the real `sleep` is no way + /// out, because uutils dispatches on argv[0] on some targets and on the + /// executed path on others. Ubuntu's `/bin/sh` is dash, which ignores its + /// argv[0]. + /// + /// `read` is a builtin, so the shell forks nothing and killing this one + /// pid leaves no orphan. If the test dies before killing it, dropping the + /// `Child` closes the pipe, `read` sees end of file, and the shell exits on + /// its own. + #[cfg(unix)] + fn spawn_decoy_engine(argv0: &Path) -> std::process::Child { + use std::os::unix::process::CommandExt; + use std::process::{Command, Stdio}; + + let mut child = Command::new("/bin/sh") + .arg0(argv0) + .args(["-c", "read -r _"]) + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("spawn the decoy engine process"); + // Long enough for a decoy that cannot run under this name to have + // exited: the multicall `sleep` above is gone in a few milliseconds. + std::thread::sleep(std::time::Duration::from_millis(250)); + let pid = child.id(); + let mut exited = child.try_wait().expect("poll the decoy engine process"); + let cmdline = std::fs::read(format!("/proc/{pid}/cmdline")).unwrap_or_default(); + let named = cmdline + .split(|&b| b == 0) + .next() + .is_some_and(|first| first == argv0.as_os_str().as_encoded_bytes()); + // Polled again when the name is wrong: a decoy that exited between the + // poll above and the read has an empty command line, and must be + // reported as having exited rather than as running under another name. + if exited.is_none() && !named { + exited = child.try_wait().expect("poll the decoy engine process"); + } + if exited.is_some() || !named { + let _ = child.kill(); + let _ = child.wait(); + } + if let Some(status) = exited { + panic!( + "the decoy engine process (pid={pid}) exited at spawn ({status}), so there \ + is nothing for --assert-clear to find; the runner's `/bin/sh` cannot run \ + under argv[0] {}. This is the fixture failing, not the gate", + argv0.display() + ); + } + assert!( + named, + "the decoy engine process (pid={pid}) is alive but its command line does not \ + start with {}, so the rule cannot match it; read: {:?}", + argv0.display(), + String::from_utf8_lossy(&cmdline) + ); + child + } + /// The preflight must run the REAL `reclaim-gpu.sh --assert-clear`, the way /// it runs it in CI, in both directions. /// @@ -2912,24 +3208,8 @@ fi // argv[0] names an E2E root and an engine marker, which is what the // rule matches, and sits under the scope so nothing else can. - // - // A copy of bash running a loop, for the reasons `spawn_decoy` in - // reclaim-gpu.sh gives: a copied multicall `sleep` exits at once as an - // unknown program, and bash would exec a lone `sleep 300` in place. - let decoy_dir = scope.path().join("e2e-prewarm/bin"); - std::fs::create_dir_all(&decoy_dir).expect("decoy dir"); - let decoy = decoy_dir.join("llama-server"); - std::fs::copy("/bin/bash", &decoy).expect("copy /bin/bash as the decoy"); - let mut child = std::process::Command::new(&decoy) - .args(["-c", "for _ in {1..300}; do sleep 1; done"]) - // Detached, as in the script: a `sleep 1` child outlives the - // SIGKILL below by up to a second and would hold the test's - // inherited stdio open meanwhile. - .stdin(std::process::Stdio::null()) - .stdout(std::process::Stdio::null()) - .stderr(std::process::Stdio::null()) - .spawn() - .expect("spawn the decoy engine process"); + let decoy = scope.path().join("e2e-prewarm/bin/llama-server"); + let mut child = spawn_decoy_engine(&decoy); let decoy_pid = child.id(); let got = run_preflight_with_reclaim( &shared_preflight_script(), @@ -2938,8 +3218,11 @@ fi &env, &reclaim, ); - // Reaped before any assertion: a panic here must not leave a 300s - // decoy behind on a self-hosted runner, of all things. + // Asked before the kill: a decoy that died during the scan would make + // a clear verdict correct, and the assertion below would blame the gate. + let alive_after_scan = matches!(child.try_wait(), Ok(None)); + // Reaped before any assertion: a panic here must not leave the decoy + // behind on a self-hosted runner, of all things. let _ = child.kill(); let _ = child.wait(); let (code, log) = got; @@ -2947,6 +3230,11 @@ fi eprintln!("skipping: the real /proc scan did not finish inside the gate's 15s bound"); return; } + assert!( + alive_after_scan, + "the decoy engine process (pid={decoy_pid}) exited while the preflight was \ + scanning, so its verdict says nothing about the gate; logged:\n{log}" + ); assert_eq!( code, 1, "a real leaked engine process was running and the preflight passed; logged:\n{log}"