diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index a22e17c0e..3b2b9eb03 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -257,11 +257,39 @@ jobs: # scenario. Persistent path; service state stays isolated per scenario. export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared" # Share uv's wheel download/build cache so `rocm install sdk` is a warm - # ~34s per scenario instead of a cold ~160s (measured on MI300X). Kept - # OFF the RUNNER_WORKSPACE Longhorn PVC (near-full) and on the roomy `/` - # overlay; ~23GB, one cold fill per pod-life. Not git-cleaned (outside - # the checkout) and outside the reclaim step's /tmp/rocm-e2e-* glob. - export E2E_SHARED_UV_CACHE_DIR="/var/tmp/rocm-e2e-uv-cache" + # ~34s per scenario instead of a cold ~160s (measured on MI300X). + # + # On RUNNER_WORKSPACE, next to the runtimes it populates, because uv can + # only hardlink out of this cache into a managed environment when the two + # are reachable without crossing a mount point. Anywhere else and it + # silently falls back to copying every file, so each runtime carries its + # own full copy of the ROCm SDK and torch stack. uv exits 0 either way and + # two of the three install paths discard its warning, so the fallback is + # invisible unless you compare inodes. + # + # ~23GB per ROCm generation, and uv never evicts, so this shares the work + # volume's budget with e2e-target/e2e-shared/e2e-prewarm rather than + # having a quota of its own. What keeps it from starving them — and from + # starving `.runner`, whose credentials live on the same volume and need a + # repo-Administration token to re-register — is the runner pod's + # `uv-cache-gc` initContainer. On the way up it sweeps abandoned `.tmp*` + # extractions, then deletes `archive-v0` outright if FREE space on the + # volume is under 60GiB. It is a free-space floor, not a size cap: the + # floor covers two full ~25GB fills, because a ROCm generation transition + # installs the new runtime beside the old one and both are resident until + # the next pod start. The work volume is ~295G, so the floor has room to + # mean something. + # + # That GC bounds ONE directory: the `uv-cache` subPath of the work PVC, + # which surfaces in this container as /uv-cache. Pointing the + # cache anywhere else on the volume keeps the hardlinks but escapes the + # floor. The 49G/49G strand that motivated all of this was a *different* + # volume — the dedicated 50Gi cache PVC, since deleted — where no floor + # existed at all; escaping this one puts the same failure back on a volume + # that also holds the runner credentials. + # $RUNNER_WORKSPACE is /_work/, so derive the root. + runner_root="$(dirname "$(dirname "$RUNNER_WORKSPACE")")" + export E2E_SHARED_UV_CACHE_DIR="$runner_root/uv-cache" # Share ONE installed managed runtime across the serve/chat scenarios so # `rocm install sdk` runs once per runner, not once per scenario (the # per-scenario install count — each a multi-GiB TheRock SDK whose probe @@ -916,10 +944,12 @@ jobs: # still warm on the next job even though the container is not. export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target" export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared" - # A SEPARATE PVC is mounted at exactly this path by the runner's cluster - # overlay. Keeping the ~23GB uv cache off the work PVC is deliberate; if - # you change this path, change the overlay that mounts it too. - export E2E_SHARED_UV_CACHE_DIR="/var/tmp/rocm-e2e-uv-cache" + # Same mount as the runtimes it populates, so uv hardlinks into them + # instead of copying, and inside the directory the runner's uv-cache-gc + # initContainer bounds — see the e2e-gpu lane for why the distinction is + # invisible in the logs and why the GC's path is load-bearing. + runner_root="$(dirname "$(dirname "$RUNNER_WORKSPACE")")" + export E2E_SHARED_UV_CACHE_DIR="$runner_root/uv-cache" # Pre-warm one shared runtime in place before the suite. Not an # optimization — `install sdk` bakes absolute paths into the runtime # manifest, so installing anywhere temporary breaks every later serve. @@ -1043,10 +1073,12 @@ jobs: # the next job even though the container is not. export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target" export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared" - # A SEPARATE volume is mounted at exactly this path by the runner's - # deployment. Keeping the ~23GB uv cache off the work volume is - # deliberate; if you change this path, change that deployment too. - export E2E_SHARED_UV_CACHE_DIR="/var/tmp/rocm-e2e-uv-cache" + # Same mount as the runtimes it populates, so uv hardlinks into them + # instead of copying, and inside the directory the runner's uv-cache-gc + # initContainer bounds — see the e2e-gpu lane for both mechanisms and + # why a cache one level deeper keeps the links but loses the floor. + runner_root="$(dirname "$(dirname "$RUNNER_WORKSPACE")")" + export E2E_SHARED_UV_CACHE_DIR="$runner_root/uv-cache" # Pre-warm one shared runtime in place before the suite. Not an # optimization — `install sdk` bakes absolute paths into the runtime # manifest, so installing anywhere temporary breaks every later serve. diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 2a18416cf..cdd40371b 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -332,10 +332,11 @@ jobs: run: gh release delete "${{ needs.nightly.outputs.staging_tag }}" --yes --cleanup-tag 2>/dev/null || true # Full E2E on app-dev MI300X INCLUDING the expensive `@nightly` scenarios (the - # large-model serve), which the per-PR ci.yml `e2e-gpu` job skips to stay fast. + # large-model serve), which the per-PR e2e-selfhosted.yml `e2e-gpu` job skips + # to stay fast. # Runs unconditionally on the schedule/dispatch (independent of the nightly # BUILD skip above — a regression run is worth doing even with no new commits). - # Non-blocking, mirrors ci.yml's e2e-gpu job but sets E2E_INCLUDE_NIGHTLY. + # Non-blocking, mirrors e2e-selfhosted.yml's e2e-gpu job but sets E2E_INCLUDE_NIGHTLY. e2e-gpu-nightly: name: E2E tests (MI300X, incl. nightly-only) timeout-minutes: 90 @@ -353,7 +354,7 @@ jobs: # Reclaim the GPU before running: kill only e2e leftovers scoped to # /tmp/rocm-e2e-* and the e2e-target/e2e-shared trees — never the runner or - # any /workload manual-testing processes (see ci.yml e2e-gpu for the same). + # any /workload manual-testing processes (see e2e-selfhosted.yml e2e-gpu for the same). - name: Reclaim GPU from stray E2E processes run: | pkill -f '/tmp/rocm-e2e.*llama-server' 2>/dev/null || true @@ -414,11 +415,15 @@ jobs: run: | export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target" export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared" - # Share uv's wheel cache off the near-full PVC, on the roomy `/` overlay - # (see ci.yml e2e-gpu): warm `rocm install sdk` ~34s vs cold ~160s. - export E2E_SHARED_UV_CACHE_DIR="/var/tmp/rocm-e2e-uv-cache" + # Share uv's wheel cache on the same mount as the runtimes it populates, + # so uv hardlinks into them instead of copying (see e2e-selfhosted.yml + # e2e-gpu): warm `rocm install sdk` ~34s vs cold ~160s. It must also be + # the directory the runner's uv-cache-gc initContainer bounds, or the + # cache grows without a floor on the volume holding `.runner`. + runner_root="$(dirname "$(dirname "$RUNNER_WORKSPACE")")" + export E2E_SHARED_UV_CACHE_DIR="$runner_root/uv-cache" # Share ONE installed managed runtime across serve/chat scenarios so - # install sdk runs once per runner, not per scenario (see ci.yml e2e-gpu). + # install sdk runs once per runner, not per scenario (see e2e-selfhosted.yml e2e-gpu). # The shared dir IS the pre-warm's own data/runtimes; NEVER move it — a # post-install mv invalidates the absolute paths install sdk bakes into # the runtime manifest and every serve fails instantly (run 29320025393). @@ -456,8 +461,8 @@ jobs: path: tests/e2e-cucumber/results/ # rad3 R9700 counterpart to e2e-gpu-nightly. Mirrors the per-PR e2e-gpu-rad3 - # lane in e2e-selfhosted.yml (scale-to-zero runner, r9700 label, own uv-cache - # PVC, GPU preflight), with @nightly scenarios enabled. Keep the two in step: + # lane in e2e-selfhosted.yml (scale-to-zero runner, r9700 label, shared uv + # cache, GPU preflight), with @nightly scenarios enabled. Keep the two in step: # the report crate asserts the nightly lanes publish the same platforms as the # per-PR lanes. e2e-gpu-nightly-rad3: @@ -521,9 +526,11 @@ jobs: # being scaled away, so the cargo target dir and shared caches stay warm. export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target" export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared" - # Separate PVC mounted at exactly this path by the runner's cluster - # overlay; if you change this path, change the overlay that mounts it too. - export E2E_SHARED_UV_CACHE_DIR="/var/tmp/rocm-e2e-uv-cache" + # Same mount as the runtimes it populates, so uv hardlinks into them + # instead of copying, and inside the uv-cache-gc initContainer's floor + # (see e2e-selfhosted.yml e2e-gpu). + runner_root="$(dirname "$(dirname "$RUNNER_WORKSPACE")")" + export E2E_SHARED_UV_CACHE_DIR="$runner_root/uv-cache" prewarm="$RUNNER_WORKSPACE/e2e-prewarm-multi-arch-v2" export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes" @@ -610,9 +617,11 @@ jobs: # away, so the cargo target dir and shared caches stay warm. export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target" export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared" - # Separate volume mounted at exactly this path by the runner's - # deployment; if you change this path, change that deployment too. - export E2E_SHARED_UV_CACHE_DIR="/var/tmp/rocm-e2e-uv-cache" + # Same mount as the runtimes it populates, so uv hardlinks into them + # instead of copying, and inside the uv-cache-gc initContainer's floor + # (see e2e-selfhosted.yml e2e-gpu). + runner_root="$(dirname "$(dirname "$RUNNER_WORKSPACE")")" + export E2E_SHARED_UV_CACHE_DIR="$runner_root/uv-cache" prewarm="$RUNNER_WORKSPACE/e2e-prewarm-multi-arch-v2" export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes" @@ -640,7 +649,8 @@ jobs: # Strix Halo counterpart to e2e-gpu-nightly. The same @nightly large-model # scenario serves the platform-specific Lemonade GGUF on this host. Keep the - # runner's storage and runtime setup aligned with ci.yml's proven Strix Ubuntu job. + # runner's storage and runtime setup aligned with e2e-selfhosted.yml's proven + # Strix Ubuntu job. e2e-gpu-nightly-strix: name: E2E tests (Strix Halo, incl. nightly-only) timeout-minutes: 90 @@ -733,8 +743,9 @@ jobs: path: tests/e2e-cucumber/results/ # Windows counterpart to e2e-gpu-nightly-strix. Keep the PowerShell-native - # setup aligned with ci.yml's proven Strix Windows job while opting into the - # same platform-adaptive @nightly large-model scenario as the Linux runner. + # setup aligned with e2e-selfhosted.yml's proven Strix Windows job while + # opting into the same platform-adaptive @nightly large-model scenario as the + # Linux runner. e2e-gpu-nightly-strix-windows: name: E2E tests (Strix Halo, Windows, incl. nightly-only) timeout-minutes: 90 diff --git a/MANIFEST.md b/MANIFEST.md index 7701e4123..2a2d71ad1 100644 --- a/MANIFEST.md +++ b/MANIFEST.md @@ -669,13 +669,15 @@ The binary is cached in the rocm-cli managed data directory and reused for subsequent operations. The version may be pinned via `ROCM_CLI_UV_VERSION`. `uv`'s own content-addressed package cache is also kept in the managed data -directory (at `/uv-cache`), so that it shares a filesystem with the -environments `uv` populates and packages can be hardlinked into them instead of -copied. This cache holds every wheel `uv` downloads — the ROCm SDK and the -torch stack included — so it is typically the largest directory rocm-cli -manages, on the order of several GB per SDK version installed. It is removed by -`rocm uninstall` unless `--keep-data` is passed, and its location can be -overridden with `ROCM_CLI_UV_CACHE_DIR`. +directory (at `/uv-cache`), so that it is reachable from the +environments `uv` populates without crossing a mount point and packages can be +hardlinked into them instead of copied. Note it is the mount, not the +filesystem: a bind mount or `subPath` volume is enough to make Linux refuse the +hardlink, and `uv` then silently copies. This cache holds every wheel `uv` +downloads — the ROCm SDK and the torch stack included — so it is typically the +largest directory rocm-cli manages, on the order of several GB per SDK version +installed. It is removed by `rocm uninstall` unless `--keep-data` is passed, and +its location can be overridden with `ROCM_CLI_UV_CACHE_DIR`. ### Lemonade Embeddable Runtime diff --git a/apps/rocm/src/therock.rs b/apps/rocm/src/therock.rs index 711cfdd9a..4dd0b13be 100644 --- a/apps/rocm/src/therock.rs +++ b/apps/rocm/src/therock.rs @@ -3541,10 +3541,12 @@ fn preflight_tarball_space( disk_space::with_margin(download_bytes), )?; - // When the cache and the install root share a filesystem, the archive and - // the extracted tree must both fit at the same time. + // When the cache and the install root share a mount, the archive and the + // extracted tree must both fit at the same time. Conservative in the other + // direction: two mounts backed by one filesystem also share a pool, and this + // treats them as separate, so the estimate can come in under the true need. let mut extract_estimate = disk_space::estimated_extracted_size(download_bytes); - if disk_space::on_same_filesystem(cache_path, install_root) == Some(true) { + if disk_space::on_same_mount(cache_path, install_root) == Some(true) { extract_estimate = extract_estimate.saturating_add(download_bytes); } Ok(disk_space::warn_if_low_space( @@ -7318,15 +7320,17 @@ mod tests { let install_root = managed_runtime_root(&paths, "wheel", runtime_key); assert!(install_root.starts_with(&paths.data_dir)); // Without --prefix the generated runtime folder is itself under the data dir, so - // the uv cache shares a filesystem with the environment it populates. + // the uv cache is reachable from the environment it populates without crossing a + // mount point. assert!(managed_uv_cache_dir(&paths.data_dir).starts_with(&paths.data_dir)); } #[test] fn uv_cache_does_not_follow_a_prefix_install_root() { // Documents a known gap rather than an intended behavior: `--prefix` relocates - // install_root only, while the uv cache stays keyed off the data dir. When the two - // land on different filesystems uv falls back to copying. Tracked separately; see + // install_root only, while the uv cache stays keyed off the data dir. When reaching + // one from the other crosses a mount point uv falls back to copying — it is the + // mount, not the filesystem, so a bind mount is enough. Tracked separately; see // the `--prefix` non-goal on the PR that introduced the colocation. let (_root, paths) = test_paths("prefix-uv-cache"); let prefix_root = PathBuf::from("/mnt/elsewhere/envs/my-env"); diff --git a/crates/rocm-core/src/disk_space.rs b/crates/rocm-core/src/disk_space.rs index 4abe85266..da5dc36fc 100644 --- a/crates/rocm-core/src/disk_space.rs +++ b/crates/rocm-core/src/disk_space.rs @@ -241,10 +241,17 @@ pub fn available_space_for_path(path: &Path) -> Option { mount_for_path(path).map(|(_, available)| available) } -/// Whether two paths live on the same filesystem. +/// Whether two paths live under the same mount point. /// -/// `None` when either path's filesystem cannot be determined. -pub fn on_same_filesystem(left: &Path, right: &Path) -> Option { +/// Named for the mount deliberately: two paths can share a filesystem and still +/// sit on different mounts (a bind mount or a `subPath` volume is enough), and +/// the mount is what the kernel enforces — `link(2)` returns `EXDEV` across two +/// mounts of one filesystem. Callers reasoning about hardlinks want this. A +/// caller reasoning about shared free space does not: those two mounts draw on +/// one pool, and this returns `false` for them. +/// +/// `None` when either path's mount cannot be determined. +pub fn on_same_mount(left: &Path, right: &Path) -> Option { let left_mount = mount_for_path(left)?.0; let right_mount = mount_for_path(right)?.0; Some(left_mount == right_mount) diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index a5226ac32..31e434e1b 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -42,7 +42,7 @@ pub use diagnose::{ pub use disk_space::{ SpaceCheck, available_space_for_path, check_space_for_path, ensure_space_for, estimated_extracted_size, format_bytes, insufficient_space_message, map_write_error, - mount_for_path, on_same_filesystem, warn_if_low_space, with_margin, + mount_for_path, on_same_mount, warn_if_low_space, with_margin, }; use examine::extract_rocm_version; pub use examine::{ diff --git a/crates/rocm-core/src/runtime.rs b/crates/rocm-core/src/runtime.rs index bb5ccd934..11afada6e 100644 --- a/crates/rocm-core/src/runtime.rs +++ b/crates/rocm-core/src/runtime.rs @@ -300,8 +300,12 @@ pub fn managed_pip_cache_dir(root: &Path) -> PathBuf { normalize_runtime_path_for_host(root).join("pip-cache") } -/// `uv`'s content-addressed cache, kept under the managed root so it shares a filesystem -/// with the environments `uv` populates and hardlinking keeps working (see issue #160). +/// `uv`'s content-addressed cache, kept under the managed root (see issue #160). +/// +/// Colocating it keeps the cache reachable from the environments `uv` populates without +/// crossing a mount point, which is what lets `uv` hardlink into them instead of copying. +/// It is the mount, not the filesystem: a bind mount or `subPath` volume is enough to make +/// Linux refuse the hardlink and send `uv` back to copying. pub fn managed_uv_cache_dir(root: &Path) -> PathBuf { normalize_runtime_path_for_host(root).join("uv-cache") } diff --git a/crates/rocm-core/src/uv.rs b/crates/rocm-core/src/uv.rs index 68bc24694..0a1fcee66 100644 --- a/crates/rocm-core/src/uv.rs +++ b/crates/rocm-core/src/uv.rs @@ -147,9 +147,11 @@ fn meaningful_cache_dir(value: Option<&OsStr>) -> Option { /// populates. /// /// Without a cache inside the managed root, `uv` caches under `$HOME/.cache/uv`; when -/// that is on a different filesystem from the data directory, `uv` cannot hardlink and +/// reaching that from the data directory crosses a mount point, `uv` cannot hardlink and /// silently copies every file, so each environment carries a full duplicate of the SDK -/// and torch stack. +/// and torch stack. Note it is the mount, not the filesystem: Linux refuses a hardlink +/// across two mounts even when both resolve to the same underlying filesystem, so a bind +/// mount or a `subPath` volume is enough to trigger the copy fallback. /// /// Note this colocates with the *data directory*, not with a `--prefix` install root; see /// the `--prefix` caveat in `docs/manual-testing.md`. diff --git a/docs/manual-testing.md b/docs/manual-testing.md index bc9f5b5b1..7a40e7e7c 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -46,12 +46,14 @@ choose a managed runtime folder and still place the pip cache inside that runtime folder at `\pip-cache`. The `uv` package cache is separate from that pip cache and does **not** follow -`--prefix`. It lives at `\uv-cache` so it shares a filesystem with the -managed environments and `uv` can hardlink into them. With `--prefix` pointing at -a different filesystem from `ROCM_CLI_DATA_DIR`, `uv` falls back to copying -packages; set `ROCM_CLI_UV_CACHE_DIR` to a folder on the prefix filesystem to -restore hardlinking. Making `--prefix` do this automatically is tracked -separately. +`--prefix`. It lives at `\uv-cache` so it is reachable from the managed +environments without crossing a mount point and `uv` can hardlink into them. With +`--prefix` pointing somewhere that is a separate mount from `ROCM_CLI_DATA_DIR`, +`uv` falls back to copying packages; set `ROCM_CLI_UV_CACHE_DIR` to a folder on +the prefix mount to restore hardlinking. Note it is the mount, not the +filesystem: a bind mount is enough to trigger the fallback even when both paths +resolve to one underlying filesystem. Making `--prefix` do this automatically is +tracked separately. ## 1. First-Time Setup diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index 80dd078a2..c1c75c83e 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -152,8 +152,11 @@ fn shared_cache_dir() -> Option { /// cold ~160s install into a ~34s warm one (measured on MI300X) without sharing /// any mutable state — the runtimes *registry* the suite asserts on still lives /// in each scenario's isolated `/runtimes` (see `default()`). Kept as its -/// own env var (not derived from `E2E_SHARED_CACHE_DIR`) so CI can place it on a -/// larger overlay disk than the model-weights cache. Unset locally → no sharing. +/// own env var (not derived from `E2E_SHARED_CACHE_DIR`) because it has a +/// placement constraint the weights cache does not: uv can only hardlink out of +/// it into a managed environment when the two are reachable without crossing a +/// mount point, so CI must put it under the same mount as the runtimes it +/// populates — the same volume is not enough. Unset locally → no sharing. fn shared_uv_cache_dir() -> Option { validated_shared_dir("E2E_SHARED_UV_CACHE_DIR") } @@ -272,8 +275,9 @@ impl E2eWorld { } // Share uv's content-addressed download/build cache (the wheels `rocm // install sdk` fetches) so only the first scenario pays the cold download. - // Independent of the weights cache above so CI can host it on a larger - // disk; the runtimes registry the suite asserts on stays isolated. + // Independent of the weights cache above because it has to sit under the + // same mount as the runtimes it populates or uv copies instead of + // hardlinking; the runtimes registry the suite asserts on stays isolated. if let Some(uv_cache) = shared_uv_cache_dir() { env.push(("UV_CACHE_DIR", uv_cache.into_os_string())); } @@ -1122,8 +1126,8 @@ async fn main() { // nightly/lifecycle, ID, and expectation resolution entirely. let only_lifecycle = std::env::var_os("E2E_ONLY_LIFECYCLE").is_some_and(|v| v == "1"); // Heavy `@merge-queue` serves run only in the merge queue (a cheaper - // per-engine canary covers them on the PR fast path); set by ci.yml on the - // `merge_group` event. + // per-engine canary covers them on the PR fast path); set by + // e2e-selfhosted.yml on the `merge_group` event. let include_merge_queue = std::env::var_os("E2E_MERGE_QUEUE").is_some_and(|v| v == "1"); eprintln!( "Host capability: platform={} os={} gpu={} effective_engine={}", diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index 5199feace..54cb6cec8 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -1106,6 +1106,74 @@ trigger-a-workflow#triggering-a-workflow-from-a-workflow" ); } + #[test] + fn every_shared_uv_cache_sits_inside_the_gc_bounded_directory() { + // Two independent properties ride on this one path, and both fail + // silently. + // + // Hardlinking: uv can only link out of its cache into a managed + // environment when the two are reachable without crossing a mount point. + // Otherwise it copies, exits 0, and two of the three install paths + // discard its warning — invisible unless you compare inodes. + // + // Eviction: the runner pod's `uv-cache-gc` initContainer bounds exactly + // one directory — the `uv-cache` subPath of the work PVC, which surfaces + // in the job as /uv-cache. It sweeps abandoned `.tmp*` + // extractions, then deletes `archive-v0` if FREE space on the volume is + // under 60GiB. A free-space floor, not a size cap. A cache placed + // elsewhere on the same volume still hardlinks, so every signal stays + // green while nothing enforces the floor — and the volume also holds + // `.runner`, whose credentials need a repo-Administration token to + // re-register. The 49G/49G strand that motivated this was the separate + // 50Gi cache PVC, since deleted, which had no floor at all. + // + // $RUNNER_WORKSPACE is /_work/, so a lane that derives + // the cache from it directly lands one level too deep and escapes the GC. + // Asserting the derivation rather than a literal keeps this honest if the + // runner root ever moves. + const DERIVATION: &str = "runner_root=\"$(dirname \"$(dirname \"$RUNNER_WORKSPACE\")\")\""; + const EXPORT: &str = "export E2E_SHARED_UV_CACHE_DIR=\"$runner_root/uv-cache\""; + + for (workflow, text) in self_hosted_workflows() { + let mut setters = 0; + for block in multiline_run_blocks(&text) { + let Some(line) = block + .lines() + .find(|line| line.starts_with("export E2E_SHARED_UV_CACHE_DIR=")) + else { + continue; + }; + setters += 1; + // Full-line equality, not `contains`: a substring match accepts + // `$runner_root/uv-cache-old`, which is outside the bounded + // directory and would fail exactly the way this test exists to + // prevent. + assert_eq!( + line, EXPORT, + "{workflow} sets the shared uv cache to `{line}`. It must be \ + exactly `{EXPORT}` — /uv-cache is the only directory \ + the runner's uv-cache-gc initContainer bounds. $RUNNER_WORKSPACE \ + itself is one level too deep, which keeps the hardlinks but \ + silently drops the 60GiB free-space floor" + ); + assert!( + block.contains(DERIVATION), + "{workflow} exports E2E_SHARED_UV_CACHE_DIR from `$runner_root` \ + without defining it in the same run block; add `{DERIVATION}`" + ); + } + // Without this, deleting every export is a silent pass — the loop + // above simply finds nothing to assert on. Same guard the sibling + // `assert_prebuilt_e2e_lanes_export_rocmd` uses. + assert!( + setters > 0, + "{workflow} sets E2E_SHARED_UV_CACHE_DIR nowhere. Its GPU lanes share a \ + pre-warmed runtime, so an unset cache sends uv to a per-job default \ + outside the GC-bounded directory" + ); + } + } + #[test] fn self_hosted_prebuilt_e2e_lanes_export_rocmd() { let workflow = read_workflow("e2e-selfhosted.yml");