diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 793329fb8..be539eb3d 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -923,11 +923,13 @@ jobs: # Windows counterpart to e2e-gpu-nightly-strix. Keep the PowerShell-native # setup aligned with e2e-selfhosted.yml's proven Strix Windows job while # opting into the same platform-adaptive @nightly large-model scenario as the - # Linux runner. + # Linux runner. Same DevLab Dispatch pool as e2e-wsl-nightly below and the + # per-PR e2e-gpu-strix-windows/e2e-wsl jobs in e2e-selfhosted.yml — there is + # no separate dedicated "native" Windows host registered for this anymore. e2e-gpu-nightly-strix-windows: name: E2E tests (Strix Halo, Windows, incl. nightly-only) timeout-minutes: 90 - runs-on: [self-hosted, windows, strix-halo, native] + runs-on: [self-hosted, windows, devlab-dispatch, strix-halo] continue-on-error: true strategy: fail-fast: false @@ -1125,8 +1127,9 @@ jobs: shell: powershell run: | # Install the shared runtime in place because its manifest contains - # absolute paths. The persistent runner workspace keeps those paths valid - # across scenarios and subsequent runs. + # absolute paths. On the ephemeral DevLab Dispatch pool this host is + # destroyed after the job, so the paths only need to stay valid for + # this run's scenarios, not across runs. $prewarm = "$env:RUNNER_WORKSPACE\e2e-prewarm-multi-arch-v2" $env:E2E_SHARED_RUNTIMES_DIR = "$prewarm\data\runtimes" @@ -1134,9 +1137,10 @@ jobs: # isolate_env() derives HF_HOME/PIP_CACHE_DIR/ROCM_CLI_CACHE_DIR from # this per scenario. Without it every serve scenario here re-downloads # its model checkpoint cold, which raced serve-08's readiness timeout - # (EAI-8031 / #260) on every nightly run — this host was never actually - # warm for it, since Windows doesn't set HOME by default and the - # engine's cache lookup didn't fall back to USERPROFILE. + # (EAI-8031 / #260) on every nightly run — on the ephemeral pool there's + # no cache carried over from a prior job, so this is cold every + # scenario, and Windows doesn't set HOME by default so the engine's + # cache lookup didn't fall back to USERPROFILE either. $env:E2E_SHARED_CACHE_DIR = "$env:RUNNER_WORKSPACE\e2e-shared" # See the e2e-gpu lane in e2e-selfhosted.yml for why the @@ -1164,12 +1168,11 @@ jobs: # WSL2 counterpart to e2e-gpu-nightly-strix: an Ubuntu-24.04 distro installed # fresh inside WSL2 on a DevLab Dispatch pool Windows host — the same pool - # the per-PR `e2e-gpu-strix-windows`/`e2e-wsl` jobs use, not a separate - # `wsl`-labeled runner and not the `native` host e2e-gpu-nightly-strix-windows - # still runs on. Mirrors the per-PR `e2e-wsl` lane in e2e-selfhosted.yml step - # for step (see that job's header comment for the pool maintainer's - # sign-off), opting into the same @nightly large-model scenario as the other - # nightly lanes. + # `e2e-gpu-nightly-strix-windows` above and the per-PR `e2e-gpu-strix-windows`/ + # `e2e-wsl` jobs use, not a separate `wsl`-labeled runner. Mirrors the per-PR + # `e2e-wsl` lane in e2e-selfhosted.yml step for step (see that job's header + # comment for the pool maintainer's sign-off), opting into the same @nightly + # large-model scenario as the other nightly lanes. e2e-wsl-nightly: name: E2E tests (Strix Halo, WSL2, incl. nightly-only) # See e2e-wsl in e2e-selfhosted.yml: every job here is cold (fresh distro, diff --git a/docs/ci-hardware-testing.md b/docs/ci-hardware-testing.md index f94da3e85..f57d6195f 100644 --- a/docs/ci-hardware-testing.md +++ b/docs/ci-hardware-testing.md @@ -58,20 +58,21 @@ against the slow DrvFs mount. The distro is deliberately left without ROCm's WSL driver bridge — see "What the WSL distro needs" below. This migration is scoped to the per-PR lanes in this table, plus the nightly -WSL lane below — not every nightly lane. `nightly.yml`'s Ubuntu and Windows -Strix lanes (`e2e-gpu-nightly-strix`, `e2e-gpu-nightly-strix-windows`) still -pin the `native` label and run on the same static, always-on host these per-PR -lanes moved off of — that label still means what it always did there: -`native` disambiguates this host from any runner that might carry a bare -`wsl` label alongside `strix-halo` (the static WSL runner this repo used -before the WSL lanes below moved to the ephemeral pool). Whether that runner -remains registered is not something this migration changes either way, since -nothing here targets a `wsl`-labeled runner anymore. `nightly.yml`'s -`e2e-wsl-nightly`, however, already moved onto the same DevLab Dispatch pool -as the per-PR `e2e-wsl` lane above, for the same reason: WSL2 needs no -dedicated `wsl`-labeled runner, so there is nothing pool-incompatible about -it. Moving the two `native` nightly lanes onto the pool as well is a -separate, not-yet-made decision. +WSL and Windows Strix lanes below — not every nightly lane. `nightly.yml`'s +Ubuntu Strix lane (`e2e-gpu-nightly-strix`) still pins the `native` label and +runs on the same static, always-on host these per-PR lanes moved off of — +that label still means what it always did there: `native` disambiguates this +host from any runner that might carry a bare `wsl` label alongside +`strix-halo` (the static WSL runner this repo used before the WSL lanes below +moved to the ephemeral pool). Whether that runner remains registered is not +something this migration changes either way, since nothing here targets a +`wsl`-labeled runner anymore. `nightly.yml`'s `e2e-wsl-nightly` and +`e2e-gpu-nightly-strix-windows`, however, both moved onto the same DevLab +Dispatch pool as the per-PR `e2e-wsl`/`e2e-gpu-strix-windows` lanes above, for +the same reason: neither needs a dedicated `native`- or `wsl`-labeled runner, +so there is nothing pool-incompatible about either. Moving the remaining +`native` Ubuntu nightly lane onto the pool as well is a separate, not-yet-made +decision. Every label in a `runs-on` must narrow the pool to one kind of hardware. In particular the MI300X lane pins `mi300x` rather than `amd-gpu`: `amd-gpu` is @@ -231,7 +232,9 @@ gh workflow run e2e-selfhosted.yml --ref -f platform=app-dev-gpu Nearly every GPU serve scenario points its `data/runtimes` at one shared, pre-warmed managed runtime tree (`E2E_SHARED_RUNTIMES_DIR`), so a multi-GiB `rocm install sdk` happens once per runner instead of once per scenario. The tree -lives on the runner's persistent workspace, survives `git clean`, and is namespaced +lives on the runner's workspace — persistent across runs on the long-lived +`native`-labeled hosts, rebuilt by the prewarm step below on each ephemeral +DevLab Dispatch pool run — survives `git clean`, and is namespaced by source-layout generation (`e2e-prewarm-multi-arch-v2`) so a branch using a new package layout cannot poison the cache consumed by code that only understands the previous layout. diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index f1067b593..a5a8db676 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -866,32 +866,35 @@ trigger-a-workflow#triggering-a-workflow-from-a-workflow" } } - /// The nightly WSL lane's `runs-on` must track the per-PR lane's, byte for - /// byte: both jobs claim to run on the same DevLab Dispatch pool host - /// (`e2e-wsl-nightly`'s header comment, docs/ci-hardware-testing.md), and - /// nothing else pins that claim -- reverting `e2e-wsl-nightly` to its old - /// static `[self-hosted, linux, strix-halo, wsl]` labels (its pre-migration - /// runs-on; `native` never applied to this job, only to the two - /// `e2e-gpu-nightly-strix*` lanes) would leave every other assertion in - /// this file green while the doc and the comment both quietly went false - /// again. + /// Each nightly DevLab Dispatch lane's `runs-on` must track its per-PR + /// counterpart's, byte for byte: both jobs in each pair claim to run on + /// the same pool host (the nightly job's header comment, + /// docs/ci-hardware-testing.md), and nothing else pins that claim -- + /// reverting a nightly job to its old static `native`/`wsl` labels would + /// leave every other assertion in this file green while the doc and the + /// comment both quietly went false again. #[test] - fn nightly_wsl_lane_shares_the_per_pr_pool_labels() { + fn nightly_devlab_dispatch_lanes_share_the_per_pr_pool_labels() { let sh = read_workflow("e2e-selfhosted.yml"); let nightly = read_workflow("nightly.yml"); - let per_pr = runs_on_values(job_block(&sh, "e2e-wsl")); - let nightly_wsl = runs_on_values(job_block(&nightly, "e2e-wsl-nightly")); - assert!(!per_pr.is_empty(), "e2e-wsl declares a runs-on"); - assert!( - per_pr.iter().any(|value| value.contains("devlab-dispatch")), - "e2e-wsl must actually be on the DevLab Dispatch pool, not just equal to \ - nightly's (equal-and-empty would pass the assertion below): {per_pr:?}" - ); - assert_eq!( - per_pr, nightly_wsl, - "e2e-wsl-nightly's runs-on must match e2e-wsl's exactly -- both are documented as \ - the same DevLab Dispatch pool" - ); + for (per_pr_job, nightly_job) in [ + ("e2e-wsl", "e2e-wsl-nightly"), + ("e2e-gpu-strix-windows", "e2e-gpu-nightly-strix-windows"), + ] { + let per_pr = runs_on_values(job_block(&sh, per_pr_job)); + let nightly_values = runs_on_values(job_block(&nightly, nightly_job)); + assert!(!per_pr.is_empty(), "{per_pr_job} declares a runs-on"); + assert!( + per_pr.iter().any(|value| value.contains("devlab-dispatch")), + "{per_pr_job} must actually be on the DevLab Dispatch pool, not just equal to \ + {nightly_job}'s (equal-and-empty would pass the assertion below): {per_pr:?}" + ); + assert_eq!( + per_pr, nightly_values, + "{nightly_job}'s runs-on must match {per_pr_job}'s exactly -- both are \ + documented as the same DevLab Dispatch pool" + ); + } } #[test]