From 9aa74516839d0aec8c248bb012e287e449c34722 Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Mon, 7 Sep 2026 12:05:11 +0000 Subject: [PATCH 1/7] ci(e2e): colocate the uv cache so uv hardlinks instead of copying The GPU lanes pointed E2E_SHARED_UV_CACHE_DIR at a fixed absolute path that is a separate mount on the self-hosted runners. uv can only hardlink out of its cache into a managed environment when the two are reachable without crossing a mount point, so every scenario's runtime was carrying its own full copy of the ROCm SDK and torch stack instead of linking to the cache. It is the mount, not the filesystem. Measured on a runner: both paths report the same st_dev and the same device, and `ln` between them still fails EXDEV, while `ln` within the work volume succeeds. A bind mount is enough to trigger the fallback even with one filesystem underneath. Nothing surfaced this. uv exits 0 whether it links or copies, and two of the three install paths capture its stderr and drop it on success, so the only way to tell them apart is to compare inodes. Moving the cache to $RUNNER_WORKSPACE puts it on the same mount as the runtime trees it populates, alongside the e2e-shared and e2e-prewarm caches that already live there. The path now has no quota of its own, so the note about eviction moves with it. Also corrects the harness and rocm-core doc comments that explained the old placement, and three "see ci.yml e2e-gpu" pointers left dangling when those lanes moved to e2e-selfhosted.yml. Signed-off-by: Tomas Saaristola --- .github/workflows/e2e-selfhosted.yml | 28 +++++++++++++++++++--------- .github/workflows/nightly.yml | 23 ++++++++++++----------- crates/rocm-core/src/uv.rs | 6 ++++-- tests/e2e-cucumber/tests/e2e.rs | 12 ++++++++---- 4 files changed, 43 insertions(+), 26 deletions(-) diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index a22e17c0e..860f88b96 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -257,11 +257,21 @@ jobs: # scenario. Persistent path; service state stays isolated per scenario. export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared" # Share uv's wheel download/build cache so `rocm install sdk` is a warm - # ~34s per scenario instead of a cold ~160s (measured on MI300X). Kept - # OFF the RUNNER_WORKSPACE Longhorn PVC (near-full) and on the roomy `/` - # overlay; ~23GB, one cold fill per pod-life. Not git-cleaned (outside - # the checkout) and outside the reclaim step's /tmp/rocm-e2e-* glob. - export E2E_SHARED_UV_CACHE_DIR="/var/tmp/rocm-e2e-uv-cache" + # ~34s per scenario instead of a cold ~160s (measured on MI300X). + # + # On RUNNER_WORKSPACE, next to the runtimes it populates, because uv can + # only hardlink out of this cache into a managed environment when the two + # are reachable without crossing a mount point. Anywhere else and it + # silently falls back to copying every file, so each runtime carries its + # own full copy of the ROCm SDK and torch stack. uv exits 0 either way and + # two of the three install paths discard its warning, so the fallback is + # invisible unless you compare inodes. + # + # ~23GB per ROCm generation, and uv never evicts, so this shares the work + # volume's budget with e2e-target/e2e-shared/e2e-prewarm rather than + # having a quota of its own. The runner image reclaims it when the volume + # runs low; nothing here does. + export E2E_SHARED_UV_CACHE_DIR="$RUNNER_WORKSPACE/uv-cache" # Share ONE installed managed runtime across the serve/chat scenarios so # `rocm install sdk` runs once per runner, not once per scenario (the # per-scenario install count — each a multi-GiB TheRock SDK whose probe @@ -916,10 +926,10 @@ jobs: # still warm on the next job even though the container is not. export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target" export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared" - # A SEPARATE PVC is mounted at exactly this path by the runner's cluster - # overlay. Keeping the ~23GB uv cache off the work PVC is deliberate; if - # you change this path, change the overlay that mounts it too. - export E2E_SHARED_UV_CACHE_DIR="/var/tmp/rocm-e2e-uv-cache" + # Same volume as the runtimes it populates, so uv hardlinks into them + # instead of copying — see the e2e-gpu lane for why the distinction is + # invisible in the logs. + export E2E_SHARED_UV_CACHE_DIR="$RUNNER_WORKSPACE/uv-cache" # Pre-warm one shared runtime in place before the suite. Not an # optimization — `install sdk` bakes absolute paths into the runtime # manifest, so installing anywhere temporary breaks every later serve. diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 2a18416cf..ebdcff1bd 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -335,7 +335,7 @@ jobs: # large-model serve), which the per-PR ci.yml `e2e-gpu` job skips to stay fast. # Runs unconditionally on the schedule/dispatch (independent of the nightly # BUILD skip above — a regression run is worth doing even with no new commits). - # Non-blocking, mirrors ci.yml's e2e-gpu job but sets E2E_INCLUDE_NIGHTLY. + # Non-blocking, mirrors e2e-selfhosted.yml's e2e-gpu job but sets E2E_INCLUDE_NIGHTLY. e2e-gpu-nightly: name: E2E tests (MI300X, incl. nightly-only) timeout-minutes: 90 @@ -353,7 +353,7 @@ jobs: # Reclaim the GPU before running: kill only e2e leftovers scoped to # /tmp/rocm-e2e-* and the e2e-target/e2e-shared trees — never the runner or - # any /workload manual-testing processes (see ci.yml e2e-gpu for the same). + # any /workload manual-testing processes (see e2e-selfhosted.yml e2e-gpu for the same). - name: Reclaim GPU from stray E2E processes run: | pkill -f '/tmp/rocm-e2e.*llama-server' 2>/dev/null || true @@ -414,11 +414,12 @@ jobs: run: | export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target" export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared" - # Share uv's wheel cache off the near-full PVC, on the roomy `/` overlay - # (see ci.yml e2e-gpu): warm `rocm install sdk` ~34s vs cold ~160s. - export E2E_SHARED_UV_CACHE_DIR="/var/tmp/rocm-e2e-uv-cache" + # Share uv's wheel cache on the same volume as the runtimes it populates, + # so uv hardlinks into them instead of copying (see e2e-selfhosted.yml + # e2e-gpu): warm `rocm install sdk` ~34s vs cold ~160s. + export E2E_SHARED_UV_CACHE_DIR="$RUNNER_WORKSPACE/uv-cache" # Share ONE installed managed runtime across serve/chat scenarios so - # install sdk runs once per runner, not per scenario (see ci.yml e2e-gpu). + # install sdk runs once per runner, not per scenario (see e2e-selfhosted.yml e2e-gpu). # The shared dir IS the pre-warm's own data/runtimes; NEVER move it — a # post-install mv invalidates the absolute paths install sdk bakes into # the runtime manifest and every serve fails instantly (run 29320025393). @@ -456,8 +457,8 @@ jobs: path: tests/e2e-cucumber/results/ # rad3 R9700 counterpart to e2e-gpu-nightly. Mirrors the per-PR e2e-gpu-rad3 - # lane in e2e-selfhosted.yml (scale-to-zero runner, r9700 label, own uv-cache - # PVC, GPU preflight), with @nightly scenarios enabled. Keep the two in step: + # lane in e2e-selfhosted.yml (scale-to-zero runner, r9700 label, shared uv + # cache, GPU preflight), with @nightly scenarios enabled. Keep the two in step: # the report crate asserts the nightly lanes publish the same platforms as the # per-PR lanes. e2e-gpu-nightly-rad3: @@ -521,9 +522,9 @@ jobs: # being scaled away, so the cargo target dir and shared caches stay warm. export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target" export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared" - # Separate PVC mounted at exactly this path by the runner's cluster - # overlay; if you change this path, change the overlay that mounts it too. - export E2E_SHARED_UV_CACHE_DIR="/var/tmp/rocm-e2e-uv-cache" + # Same volume as the runtimes it populates, so uv hardlinks into them + # instead of copying (see e2e-selfhosted.yml e2e-gpu). + export E2E_SHARED_UV_CACHE_DIR="$RUNNER_WORKSPACE/uv-cache" prewarm="$RUNNER_WORKSPACE/e2e-prewarm-multi-arch-v2" export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes" diff --git a/crates/rocm-core/src/uv.rs b/crates/rocm-core/src/uv.rs index 68bc24694..0a1fcee66 100644 --- a/crates/rocm-core/src/uv.rs +++ b/crates/rocm-core/src/uv.rs @@ -147,9 +147,11 @@ fn meaningful_cache_dir(value: Option<&OsStr>) -> Option { /// populates. /// /// Without a cache inside the managed root, `uv` caches under `$HOME/.cache/uv`; when -/// that is on a different filesystem from the data directory, `uv` cannot hardlink and +/// reaching that from the data directory crosses a mount point, `uv` cannot hardlink and /// silently copies every file, so each environment carries a full duplicate of the SDK -/// and torch stack. +/// and torch stack. Note it is the mount, not the filesystem: Linux refuses a hardlink +/// across two mounts even when both resolve to the same underlying filesystem, so a bind +/// mount or a `subPath` volume is enough to trigger the copy fallback. /// /// Note this colocates with the *data directory*, not with a `--prefix` install root; see /// the `--prefix` caveat in `docs/manual-testing.md`. diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index 80dd078a2..547c294cb 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -152,8 +152,11 @@ fn shared_cache_dir() -> Option { /// cold ~160s install into a ~34s warm one (measured on MI300X) without sharing /// any mutable state — the runtimes *registry* the suite asserts on still lives /// in each scenario's isolated `/runtimes` (see `default()`). Kept as its -/// own env var (not derived from `E2E_SHARED_CACHE_DIR`) so CI can place it on a -/// larger overlay disk than the model-weights cache. Unset locally → no sharing. +/// own env var (not derived from `E2E_SHARED_CACHE_DIR`) because it has a +/// placement constraint the weights cache does not: uv can only hardlink out of +/// it into a managed environment when the two are reachable without crossing a +/// mount point, so CI must put it on the same volume as the runtimes it +/// populates. Unset locally → no sharing. fn shared_uv_cache_dir() -> Option { validated_shared_dir("E2E_SHARED_UV_CACHE_DIR") } @@ -272,8 +275,9 @@ impl E2eWorld { } // Share uv's content-addressed download/build cache (the wheels `rocm // install sdk` fetches) so only the first scenario pays the cold download. - // Independent of the weights cache above so CI can host it on a larger - // disk; the runtimes registry the suite asserts on stays isolated. + // Independent of the weights cache above because it has to sit on the + // same volume as the runtimes it populates or uv copies instead of + // hardlinking; the runtimes registry the suite asserts on stays isolated. if let Some(uv_cache) = shared_uv_cache_dir() { env.push(("UV_CACHE_DIR", uv_cache.into_os_string())); } From 39005a08aa011b133fe21c4b007506fa874dcd1d Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Wed, 9 Sep 2026 14:53:01 +0000 Subject: [PATCH 2/7] docs(ci): finish the mount-vs-filesystem and workflow-pointer sweep The parent commit corrected the uv-cache placement rationale in uv.rs and the E2E harness but left the same claim standing in two other places, and repaired only some of the pointers left dangling when the self-hosted GPU lanes moved out of ci.yml. Sweep the rest: docs/manual-testing.md and the therock.rs `--prefix` tests said "different filesystem" when the constraint is the mount (Linux refuses a cross-mount hardlink even within one filesystem), and four `ci.yml` references now resolve to e2e-selfhosted.yml -- including the one on E2E_MERGE_QUEUE, which only that workflow sets. Comments and prose only; no executable line changes. Signed-off-by: Tomas Saaristola --- .github/workflows/nightly.yml | 11 +++++++---- apps/rocm/src/therock.rs | 8 +++++--- docs/manual-testing.md | 14 ++++++++------ tests/e2e-cucumber/tests/e2e.rs | 4 ++-- 4 files changed, 22 insertions(+), 15 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index ebdcff1bd..6843bc2f0 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -332,7 +332,8 @@ jobs: run: gh release delete "${{ needs.nightly.outputs.staging_tag }}" --yes --cleanup-tag 2>/dev/null || true # Full E2E on app-dev MI300X INCLUDING the expensive `@nightly` scenarios (the - # large-model serve), which the per-PR ci.yml `e2e-gpu` job skips to stay fast. + # large-model serve), which the per-PR e2e-selfhosted.yml `e2e-gpu` job skips + # to stay fast. # Runs unconditionally on the schedule/dispatch (independent of the nightly # BUILD skip above — a regression run is worth doing even with no new commits). # Non-blocking, mirrors e2e-selfhosted.yml's e2e-gpu job but sets E2E_INCLUDE_NIGHTLY. @@ -641,7 +642,8 @@ jobs: # Strix Halo counterpart to e2e-gpu-nightly. The same @nightly large-model # scenario serves the platform-specific Lemonade GGUF on this host. Keep the - # runner's storage and runtime setup aligned with ci.yml's proven Strix Ubuntu job. + # runner's storage and runtime setup aligned with e2e-selfhosted.yml's proven + # Strix Ubuntu job. e2e-gpu-nightly-strix: name: E2E tests (Strix Halo, incl. nightly-only) timeout-minutes: 90 @@ -734,8 +736,9 @@ jobs: path: tests/e2e-cucumber/results/ # Windows counterpart to e2e-gpu-nightly-strix. Keep the PowerShell-native - # setup aligned with ci.yml's proven Strix Windows job while opting into the - # same platform-adaptive @nightly large-model scenario as the Linux runner. + # setup aligned with e2e-selfhosted.yml's proven Strix Windows job while + # opting into the same platform-adaptive @nightly large-model scenario as the + # Linux runner. e2e-gpu-nightly-strix-windows: name: E2E tests (Strix Halo, Windows, incl. nightly-only) timeout-minutes: 90 diff --git a/apps/rocm/src/therock.rs b/apps/rocm/src/therock.rs index 711cfdd9a..09d15a8fc 100644 --- a/apps/rocm/src/therock.rs +++ b/apps/rocm/src/therock.rs @@ -7318,15 +7318,17 @@ mod tests { let install_root = managed_runtime_root(&paths, "wheel", runtime_key); assert!(install_root.starts_with(&paths.data_dir)); // Without --prefix the generated runtime folder is itself under the data dir, so - // the uv cache shares a filesystem with the environment it populates. + // the uv cache is reachable from the environment it populates without crossing a + // mount point. assert!(managed_uv_cache_dir(&paths.data_dir).starts_with(&paths.data_dir)); } #[test] fn uv_cache_does_not_follow_a_prefix_install_root() { // Documents a known gap rather than an intended behavior: `--prefix` relocates - // install_root only, while the uv cache stays keyed off the data dir. When the two - // land on different filesystems uv falls back to copying. Tracked separately; see + // install_root only, while the uv cache stays keyed off the data dir. When reaching + // one from the other crosses a mount point uv falls back to copying — it is the + // mount, not the filesystem, so a bind mount is enough. Tracked separately; see // the `--prefix` non-goal on the PR that introduced the colocation. let (_root, paths) = test_paths("prefix-uv-cache"); let prefix_root = PathBuf::from("/mnt/elsewhere/envs/my-env"); diff --git a/docs/manual-testing.md b/docs/manual-testing.md index bc9f5b5b1..7a40e7e7c 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -46,12 +46,14 @@ choose a managed runtime folder and still place the pip cache inside that runtime folder at `\pip-cache`. The `uv` package cache is separate from that pip cache and does **not** follow -`--prefix`. It lives at `\uv-cache` so it shares a filesystem with the -managed environments and `uv` can hardlink into them. With `--prefix` pointing at -a different filesystem from `ROCM_CLI_DATA_DIR`, `uv` falls back to copying -packages; set `ROCM_CLI_UV_CACHE_DIR` to a folder on the prefix filesystem to -restore hardlinking. Making `--prefix` do this automatically is tracked -separately. +`--prefix`. It lives at `\uv-cache` so it is reachable from the managed +environments without crossing a mount point and `uv` can hardlink into them. With +`--prefix` pointing somewhere that is a separate mount from `ROCM_CLI_DATA_DIR`, +`uv` falls back to copying packages; set `ROCM_CLI_UV_CACHE_DIR` to a folder on +the prefix mount to restore hardlinking. Note it is the mount, not the +filesystem: a bind mount is enough to trigger the fallback even when both paths +resolve to one underlying filesystem. Making `--prefix` do this automatically is +tracked separately. ## 1. First-Time Setup diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index 547c294cb..2299610b2 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -1126,8 +1126,8 @@ async fn main() { // nightly/lifecycle, ID, and expectation resolution entirely. let only_lifecycle = std::env::var_os("E2E_ONLY_LIFECYCLE").is_some_and(|v| v == "1"); // Heavy `@merge-queue` serves run only in the merge queue (a cheaper - // per-engine canary covers them on the PR fast path); set by ci.yml on the - // `merge_group` event. + // per-engine canary covers them on the PR fast path); set by + // e2e-selfhosted.yml on the `merge_group` event. let include_merge_queue = std::env::var_os("E2E_MERGE_QUEUE").is_some_and(|v| v == "1"); eprintln!( "Host capability: platform={} os={} gpu={} effective_engine={}", From fccd64428eae3b7aabb9386c33d1287d9e0fbe2b Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Thu, 10 Sep 2026 10:37:35 +0000 Subject: [PATCH 3/7] docs: correct the last two mount-vs-filesystem claims The sweep missed the canonical spot: the doc comment on managed_uv_cache_dir, the function that implements the colocation, plus its MANIFEST.md counterpart. Both still claimed a shared *filesystem* was what enables hardlinking. Signed-off-by: Tomas Saaristola --- MANIFEST.md | 9 ++++++--- crates/rocm-core/src/runtime.rs | 6 ++++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/MANIFEST.md b/MANIFEST.md index 7701e4123..cbfb36767 100644 --- a/MANIFEST.md +++ b/MANIFEST.md @@ -669,9 +669,12 @@ The binary is cached in the rocm-cli managed data directory and reused for subsequent operations. The version may be pinned via `ROCM_CLI_UV_VERSION`. `uv`'s own content-addressed package cache is also kept in the managed data -directory (at `/uv-cache`), so that it shares a filesystem with the -environments `uv` populates and packages can be hardlinked into them instead of -copied. This cache holds every wheel `uv` downloads — the ROCm SDK and the +directory (at `/uv-cache`), so that it is reachable from the +environments `uv` populates without crossing a mount point and packages can be +hardlinked into them instead of copied. Note it is the mount, not the +filesystem: a bind mount or `subPath` volume is enough to make Linux refuse the +hardlink, and `uv` then silently copies. This cache holds every wheel `uv` +downloads — the ROCm SDK and the torch stack included — so it is typically the largest directory rocm-cli manages, on the order of several GB per SDK version installed. It is removed by `rocm uninstall` unless `--keep-data` is passed, and its location can be diff --git a/crates/rocm-core/src/runtime.rs b/crates/rocm-core/src/runtime.rs index bb5ccd934..7e2c24078 100644 --- a/crates/rocm-core/src/runtime.rs +++ b/crates/rocm-core/src/runtime.rs @@ -300,8 +300,10 @@ pub fn managed_pip_cache_dir(root: &Path) -> PathBuf { normalize_runtime_path_for_host(root).join("pip-cache") } -/// `uv`'s content-addressed cache, kept under the managed root so it shares a filesystem -/// with the environments `uv` populates and hardlinking keeps working (see issue #160). +/// `uv`'s content-addressed cache, kept under the managed root so it is reachable from the +/// environments `uv` populates without crossing a mount point and hardlinking keeps working +/// (see issue #160). It is the mount, not the filesystem: a bind mount or `subPath` volume +/// is enough to make Linux refuse the hardlink and send `uv` back to copying. pub fn managed_uv_cache_dir(root: &Path) -> PathBuf { normalize_runtime_path_for_host(root).join("uv-cache") } From d4c7fc0d502fb70267e69db31ae547cefe33dc95 Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Mon, 14 Sep 2026 11:24:43 +0000 Subject: [PATCH 4/7] ci(e2e): point the shared uv cache at the GC-bounded directory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The colocation moved the cache to \$RUNNER_WORKSPACE/uv-cache, which restores hardlinking but lands one level below the only directory the runner pod's uv-cache-gc initContainer bounds. RUNNER_WORKSPACE is /_work/, measured on a live MI300X job. The GC mounts the work PVC's `uv-cache` subPath and sweeps /uv-cache: abandoned .tmp* extractions, and archive-v0 once free space drops under a 60GiB floor. A cache one level deeper still hardlinks, so every signal stays green while nothing enforces the floor — on the volume that also holds .runner, whose credentials need a repo-Administration token to re-register. That is how the MI300X lane previously stranded at 49G/49G. Derive the runner root instead of hardcoding it, and assert the derivation in workflow_contract so the next lane cannot reintroduce it: reverting any one of the four exports now fails the suite, which it did not before. Signed-off-by: Tomas Saaristola --- .github/workflows/e2e-selfhosted.yml | 26 ++++++++++---- .github/workflows/nightly.yml | 17 ++++++---- xtask/src/workflow_contract.rs | 51 ++++++++++++++++++++++++++++ 3 files changed, 81 insertions(+), 13 deletions(-) diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index 860f88b96..fb0563b32 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -269,9 +269,19 @@ jobs: # # ~23GB per ROCm generation, and uv never evicts, so this shares the work # volume's budget with e2e-target/e2e-shared/e2e-prewarm rather than - # having a quota of its own. The runner image reclaims it when the volume - # runs low; nothing here does. - export E2E_SHARED_UV_CACHE_DIR="$RUNNER_WORKSPACE/uv-cache" + # having a quota of its own. What keeps it from starving them — and from + # starving `.runner`, whose credentials live on the same volume and need a + # repo-Administration token to re-register — is the runner pod's + # `uv-cache-gc` initContainer, which sweeps abandoned `.tmp*` extractions + # and drops `archive-v0` below a 60GiB floor on the way up. + # + # That GC bounds ONE directory: the `uv-cache` subPath of the work PVC, + # which surfaces in this container as /uv-cache. Pointing the + # cache anywhere else on the volume keeps the hardlinks but escapes the + # floor, which is how the MI300X lane previously stranded at 49G/49G. + # $RUNNER_WORKSPACE is /_work/, so derive the root. + runner_root="$(dirname "$(dirname "$RUNNER_WORKSPACE")")" + export E2E_SHARED_UV_CACHE_DIR="$runner_root/uv-cache" # Share ONE installed managed runtime across the serve/chat scenarios so # `rocm install sdk` runs once per runner, not once per scenario (the # per-scenario install count — each a multi-GiB TheRock SDK whose probe @@ -926,10 +936,12 @@ jobs: # still warm on the next job even though the container is not. export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target" export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared" - # Same volume as the runtimes it populates, so uv hardlinks into them - # instead of copying — see the e2e-gpu lane for why the distinction is - # invisible in the logs. - export E2E_SHARED_UV_CACHE_DIR="$RUNNER_WORKSPACE/uv-cache" + # Same mount as the runtimes it populates, so uv hardlinks into them + # instead of copying, and inside the directory the runner's uv-cache-gc + # initContainer bounds — see the e2e-gpu lane for why the distinction is + # invisible in the logs and why the GC's path is load-bearing. + runner_root="$(dirname "$(dirname "$RUNNER_WORKSPACE")")" + export E2E_SHARED_UV_CACHE_DIR="$runner_root/uv-cache" # Pre-warm one shared runtime in place before the suite. Not an # optimization — `install sdk` bakes absolute paths into the runtime # manifest, so installing anywhere temporary breaks every later serve. diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 6843bc2f0..669edbb27 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -415,10 +415,13 @@ jobs: run: | export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target" export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared" - # Share uv's wheel cache on the same volume as the runtimes it populates, + # Share uv's wheel cache on the same mount as the runtimes it populates, # so uv hardlinks into them instead of copying (see e2e-selfhosted.yml - # e2e-gpu): warm `rocm install sdk` ~34s vs cold ~160s. - export E2E_SHARED_UV_CACHE_DIR="$RUNNER_WORKSPACE/uv-cache" + # e2e-gpu): warm `rocm install sdk` ~34s vs cold ~160s. It must also be + # the directory the runner's uv-cache-gc initContainer bounds, or the + # cache grows without a floor on the volume holding `.runner`. + runner_root="$(dirname "$(dirname "$RUNNER_WORKSPACE")")" + export E2E_SHARED_UV_CACHE_DIR="$runner_root/uv-cache" # Share ONE installed managed runtime across serve/chat scenarios so # install sdk runs once per runner, not per scenario (see e2e-selfhosted.yml e2e-gpu). # The shared dir IS the pre-warm's own data/runtimes; NEVER move it — a @@ -523,9 +526,11 @@ jobs: # being scaled away, so the cargo target dir and shared caches stay warm. export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target" export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared" - # Same volume as the runtimes it populates, so uv hardlinks into them - # instead of copying (see e2e-selfhosted.yml e2e-gpu). - export E2E_SHARED_UV_CACHE_DIR="$RUNNER_WORKSPACE/uv-cache" + # Same mount as the runtimes it populates, so uv hardlinks into them + # instead of copying, and inside the uv-cache-gc initContainer's floor + # (see e2e-selfhosted.yml e2e-gpu). + runner_root="$(dirname "$(dirname "$RUNNER_WORKSPACE")")" + export E2E_SHARED_UV_CACHE_DIR="$runner_root/uv-cache" prewarm="$RUNNER_WORKSPACE/e2e-prewarm-multi-arch-v2" export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes" diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index 5199feace..748832998 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -1106,6 +1106,57 @@ trigger-a-workflow#triggering-a-workflow-from-a-workflow" ); } + #[test] + fn every_shared_uv_cache_sits_inside_the_gc_bounded_directory() { + // Two independent properties ride on this one path, and both fail + // silently. + // + // Hardlinking: uv can only link out of its cache into a managed + // environment when the two are reachable without crossing a mount point. + // Otherwise it copies, exits 0, and two of the three install paths + // discard its warning — invisible unless you compare inodes. + // + // Eviction: the runner pod's `uv-cache-gc` initContainer bounds exactly + // one directory — the `uv-cache` subPath of the work PVC, which surfaces + // in the job as /uv-cache. It sweeps abandoned `.tmp*` + // extractions and drops `archive-v0` below a 60GiB floor. A cache placed + // elsewhere on the same volume still hardlinks, so every signal stays + // green while nothing enforces the floor — and the volume also holds + // `.runner`, whose credentials need a repo-Administration token to + // re-register. That is how the MI300X lane stranded at 49G/49G. + // + // $RUNNER_WORKSPACE is /_work/, so a lane that derives + // the cache from it directly lands one level too deep and escapes the GC. + // Asserting the derivation rather than a literal keeps this honest if the + // runner root ever moves. + for (workflow, text) in self_hosted_workflows() { + for block in multiline_run_blocks(&text) { + let Some(line) = block + .lines() + .find(|line| line.starts_with("export E2E_SHARED_UV_CACHE_DIR=")) + else { + continue; + }; + assert!( + line.contains("$runner_root/uv-cache"), + "{workflow} sets the shared uv cache to `{line}`. It must be \ + /uv-cache — the only directory the runner's \ + uv-cache-gc initContainer bounds. Derive it with \ + `runner_root=\"$(dirname \"$(dirname \"$RUNNER_WORKSPACE\")\")\"`; \ + $RUNNER_WORKSPACE itself is one level too deep, which keeps the \ + hardlinks but silently drops the 60GiB floor" + ); + assert!( + block.contains( + "runner_root=\"$(dirname \"$(dirname \"$RUNNER_WORKSPACE\")\")\"" + ), + "{workflow} exports E2E_SHARED_UV_CACHE_DIR from `$runner_root` \ + without defining it in the same run block" + ); + } + } + } + #[test] fn self_hosted_prebuilt_e2e_lanes_export_rocmd() { let workflow = read_workflow("e2e-selfhosted.yml"); From 0399b32e0437a66d352564b761ee7e1bfdb53319 Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Mon, 14 Sep 2026 12:22:31 +0000 Subject: [PATCH 5/7] docs: split the managed_uv_cache_dir doc comment clippy's too_long_first_doc_paragraph is denied workspace-wide, and the reworded comment ran the summary and the mount-vs-filesystem detail together into one four-line paragraph. Same content, summary sentence first. `cargo check` does not see this; `cargo clippy -D warnings` does. Signed-off-by: Tomas Saaristola --- crates/rocm-core/src/runtime.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/crates/rocm-core/src/runtime.rs b/crates/rocm-core/src/runtime.rs index 7e2c24078..11afada6e 100644 --- a/crates/rocm-core/src/runtime.rs +++ b/crates/rocm-core/src/runtime.rs @@ -300,10 +300,12 @@ pub fn managed_pip_cache_dir(root: &Path) -> PathBuf { normalize_runtime_path_for_host(root).join("pip-cache") } -/// `uv`'s content-addressed cache, kept under the managed root so it is reachable from the -/// environments `uv` populates without crossing a mount point and hardlinking keeps working -/// (see issue #160). It is the mount, not the filesystem: a bind mount or `subPath` volume -/// is enough to make Linux refuse the hardlink and send `uv` back to copying. +/// `uv`'s content-addressed cache, kept under the managed root (see issue #160). +/// +/// Colocating it keeps the cache reachable from the environments `uv` populates without +/// crossing a mount point, which is what lets `uv` hardlink into them instead of copying. +/// It is the mount, not the filesystem: a bind mount or `subPath` volume is enough to make +/// Linux refuse the hardlink and send `uv` back to copying. pub fn managed_uv_cache_dir(root: &Path) -> PathBuf { normalize_runtime_path_for_host(root).join("uv-cache") } From f2f81f5bfdbe76ff54fc8982408e9a87964b7325 Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Mon, 14 Sep 2026 12:29:16 +0000 Subject: [PATCH 6/7] ci(e2e): state the GC mechanism precisely, and harden its contract test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The comment said the GC "drops archive-v0 below a 60GiB floor" while the commit message said it fires "once free space drops under" one. Those are different mechanisms, and the source settles it: uv-cache-gc reads column 4 of `df -Pk` (Available) and deletes archive-v0 outright when it is under 60GiB. A free-space floor, not a size cap. Say so identically in the workflow comment and the xtask test. The 49G/49G figure sat next to the 60GiB floor implying one volume, where it would be either unsatisfiable or inert. They are different volumes: the strand was the dedicated 50Gi cache PVC, since deleted; the floor applies to the ~295G work volume. Say which is which. Test hardening, both cases the assertions let through: - full-line equality instead of `contains`, which accepted `$runner_root/uv-cache-old` — outside the bounded directory - a per-workflow setter count, so deleting every export fails instead of passing an empty loop, matching assert_prebuilt_e2e_lanes_export_rocmd Also rename on_same_filesystem -> on_same_mount. It compares mount points and always did; the name taught the exact wrong model this PR exists to correct, in the one API that embodies it. Its space-check caller now notes the conservative direction: two mounts over one filesystem do share a pool. Signed-off-by: Tomas Saaristola --- .github/workflows/e2e-selfhosted.yml | 14 +++++++-- MANIFEST.md | 9 +++--- apps/rocm/src/therock.rs | 8 ++++-- crates/rocm-core/src/disk_space.rs | 13 +++++++-- crates/rocm-core/src/lib.rs | 2 +- tests/e2e-cucumber/tests/e2e.rs | 8 +++--- xtask/src/workflow_contract.rs | 43 +++++++++++++++++++--------- 7 files changed, 65 insertions(+), 32 deletions(-) diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index fb0563b32..4b58abc9c 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -272,13 +272,21 @@ jobs: # having a quota of its own. What keeps it from starving them — and from # starving `.runner`, whose credentials live on the same volume and need a # repo-Administration token to re-register — is the runner pod's - # `uv-cache-gc` initContainer, which sweeps abandoned `.tmp*` extractions - # and drops `archive-v0` below a 60GiB floor on the way up. + # `uv-cache-gc` initContainer. On the way up it sweeps abandoned `.tmp*` + # extractions, then deletes `archive-v0` outright if FREE space on the + # volume is under 60GiB. It is a free-space floor, not a size cap: the + # floor covers two full ~25GB fills, because a ROCm generation transition + # installs the new runtime beside the old one and both are resident until + # the next pod start. The work volume is ~295G, so the floor has room to + # mean something. # # That GC bounds ONE directory: the `uv-cache` subPath of the work PVC, # which surfaces in this container as /uv-cache. Pointing the # cache anywhere else on the volume keeps the hardlinks but escapes the - # floor, which is how the MI300X lane previously stranded at 49G/49G. + # floor. The 49G/49G strand that motivated all of this was a *different* + # volume — the dedicated 50Gi cache PVC, since deleted — where no floor + # existed at all; escaping this one puts the same failure back on a volume + # that also holds the runner credentials. # $RUNNER_WORKSPACE is /_work/, so derive the root. runner_root="$(dirname "$(dirname "$RUNNER_WORKSPACE")")" export E2E_SHARED_UV_CACHE_DIR="$runner_root/uv-cache" diff --git a/MANIFEST.md b/MANIFEST.md index cbfb36767..2a2d71ad1 100644 --- a/MANIFEST.md +++ b/MANIFEST.md @@ -674,11 +674,10 @@ environments `uv` populates without crossing a mount point and packages can be hardlinked into them instead of copied. Note it is the mount, not the filesystem: a bind mount or `subPath` volume is enough to make Linux refuse the hardlink, and `uv` then silently copies. This cache holds every wheel `uv` -downloads — the ROCm SDK and the -torch stack included — so it is typically the largest directory rocm-cli -manages, on the order of several GB per SDK version installed. It is removed by -`rocm uninstall` unless `--keep-data` is passed, and its location can be -overridden with `ROCM_CLI_UV_CACHE_DIR`. +downloads — the ROCm SDK and the torch stack included — so it is typically the +largest directory rocm-cli manages, on the order of several GB per SDK version +installed. It is removed by `rocm uninstall` unless `--keep-data` is passed, and +its location can be overridden with `ROCM_CLI_UV_CACHE_DIR`. ### Lemonade Embeddable Runtime diff --git a/apps/rocm/src/therock.rs b/apps/rocm/src/therock.rs index 09d15a8fc..4dd0b13be 100644 --- a/apps/rocm/src/therock.rs +++ b/apps/rocm/src/therock.rs @@ -3541,10 +3541,12 @@ fn preflight_tarball_space( disk_space::with_margin(download_bytes), )?; - // When the cache and the install root share a filesystem, the archive and - // the extracted tree must both fit at the same time. + // When the cache and the install root share a mount, the archive and the + // extracted tree must both fit at the same time. Conservative in the other + // direction: two mounts backed by one filesystem also share a pool, and this + // treats them as separate, so the estimate can come in under the true need. let mut extract_estimate = disk_space::estimated_extracted_size(download_bytes); - if disk_space::on_same_filesystem(cache_path, install_root) == Some(true) { + if disk_space::on_same_mount(cache_path, install_root) == Some(true) { extract_estimate = extract_estimate.saturating_add(download_bytes); } Ok(disk_space::warn_if_low_space( diff --git a/crates/rocm-core/src/disk_space.rs b/crates/rocm-core/src/disk_space.rs index 4abe85266..da5dc36fc 100644 --- a/crates/rocm-core/src/disk_space.rs +++ b/crates/rocm-core/src/disk_space.rs @@ -241,10 +241,17 @@ pub fn available_space_for_path(path: &Path) -> Option { mount_for_path(path).map(|(_, available)| available) } -/// Whether two paths live on the same filesystem. +/// Whether two paths live under the same mount point. /// -/// `None` when either path's filesystem cannot be determined. -pub fn on_same_filesystem(left: &Path, right: &Path) -> Option { +/// Named for the mount deliberately: two paths can share a filesystem and still +/// sit on different mounts (a bind mount or a `subPath` volume is enough), and +/// the mount is what the kernel enforces — `link(2)` returns `EXDEV` across two +/// mounts of one filesystem. Callers reasoning about hardlinks want this. A +/// caller reasoning about shared free space does not: those two mounts draw on +/// one pool, and this returns `false` for them. +/// +/// `None` when either path's mount cannot be determined. +pub fn on_same_mount(left: &Path, right: &Path) -> Option { let left_mount = mount_for_path(left)?.0; let right_mount = mount_for_path(right)?.0; Some(left_mount == right_mount) diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index a5226ac32..31e434e1b 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -42,7 +42,7 @@ pub use diagnose::{ pub use disk_space::{ SpaceCheck, available_space_for_path, check_space_for_path, ensure_space_for, estimated_extracted_size, format_bytes, insufficient_space_message, map_write_error, - mount_for_path, on_same_filesystem, warn_if_low_space, with_margin, + mount_for_path, on_same_mount, warn_if_low_space, with_margin, }; use examine::extract_rocm_version; pub use examine::{ diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index 2299610b2..c1c75c83e 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -155,8 +155,8 @@ fn shared_cache_dir() -> Option { /// own env var (not derived from `E2E_SHARED_CACHE_DIR`) because it has a /// placement constraint the weights cache does not: uv can only hardlink out of /// it into a managed environment when the two are reachable without crossing a -/// mount point, so CI must put it on the same volume as the runtimes it -/// populates. Unset locally → no sharing. +/// mount point, so CI must put it under the same mount as the runtimes it +/// populates — the same volume is not enough. Unset locally → no sharing. fn shared_uv_cache_dir() -> Option { validated_shared_dir("E2E_SHARED_UV_CACHE_DIR") } @@ -275,8 +275,8 @@ impl E2eWorld { } // Share uv's content-addressed download/build cache (the wheels `rocm // install sdk` fetches) so only the first scenario pays the cold download. - // Independent of the weights cache above because it has to sit on the - // same volume as the runtimes it populates or uv copies instead of + // Independent of the weights cache above because it has to sit under the + // same mount as the runtimes it populates or uv copies instead of // hardlinking; the runtimes registry the suite asserts on stays isolated. if let Some(uv_cache) = shared_uv_cache_dir() { env.push(("UV_CACHE_DIR", uv_cache.into_os_string())); diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index 748832998..54cb6cec8 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -1119,17 +1119,23 @@ trigger-a-workflow#triggering-a-workflow-from-a-workflow" // Eviction: the runner pod's `uv-cache-gc` initContainer bounds exactly // one directory — the `uv-cache` subPath of the work PVC, which surfaces // in the job as /uv-cache. It sweeps abandoned `.tmp*` - // extractions and drops `archive-v0` below a 60GiB floor. A cache placed + // extractions, then deletes `archive-v0` if FREE space on the volume is + // under 60GiB. A free-space floor, not a size cap. A cache placed // elsewhere on the same volume still hardlinks, so every signal stays // green while nothing enforces the floor — and the volume also holds // `.runner`, whose credentials need a repo-Administration token to - // re-register. That is how the MI300X lane stranded at 49G/49G. + // re-register. The 49G/49G strand that motivated this was the separate + // 50Gi cache PVC, since deleted, which had no floor at all. // // $RUNNER_WORKSPACE is /_work/, so a lane that derives // the cache from it directly lands one level too deep and escapes the GC. // Asserting the derivation rather than a literal keeps this honest if the // runner root ever moves. + const DERIVATION: &str = "runner_root=\"$(dirname \"$(dirname \"$RUNNER_WORKSPACE\")\")\""; + const EXPORT: &str = "export E2E_SHARED_UV_CACHE_DIR=\"$runner_root/uv-cache\""; + for (workflow, text) in self_hosted_workflows() { + let mut setters = 0; for block in multiline_run_blocks(&text) { let Some(line) = block .lines() @@ -1137,23 +1143,34 @@ trigger-a-workflow#triggering-a-workflow-from-a-workflow" else { continue; }; - assert!( - line.contains("$runner_root/uv-cache"), + setters += 1; + // Full-line equality, not `contains`: a substring match accepts + // `$runner_root/uv-cache-old`, which is outside the bounded + // directory and would fail exactly the way this test exists to + // prevent. + assert_eq!( + line, EXPORT, "{workflow} sets the shared uv cache to `{line}`. It must be \ - /uv-cache — the only directory the runner's \ - uv-cache-gc initContainer bounds. Derive it with \ - `runner_root=\"$(dirname \"$(dirname \"$RUNNER_WORKSPACE\")\")\"`; \ - $RUNNER_WORKSPACE itself is one level too deep, which keeps the \ - hardlinks but silently drops the 60GiB floor" + exactly `{EXPORT}` — /uv-cache is the only directory \ + the runner's uv-cache-gc initContainer bounds. $RUNNER_WORKSPACE \ + itself is one level too deep, which keeps the hardlinks but \ + silently drops the 60GiB free-space floor" ); assert!( - block.contains( - "runner_root=\"$(dirname \"$(dirname \"$RUNNER_WORKSPACE\")\")\"" - ), + block.contains(DERIVATION), "{workflow} exports E2E_SHARED_UV_CACHE_DIR from `$runner_root` \ - without defining it in the same run block" + without defining it in the same run block; add `{DERIVATION}`" ); } + // Without this, deleting every export is a silent pass — the loop + // above simply finds nothing to assert on. Same guard the sibling + // `assert_prebuilt_e2e_lanes_export_rocmd` uses. + assert!( + setters > 0, + "{workflow} sets E2E_SHARED_UV_CACHE_DIR nowhere. Its GPU lanes share a \ + pre-warmed runtime, so an unset cache sends uv to a per-job default \ + outside the GC-bounded directory" + ); } } From 20aed9758ceb0b7d2497fb337bb32a9891acf90d Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Mon, 14 Sep 2026 12:45:19 +0000 Subject: [PATCH 7/7] ci(e2e): bring the MI350P lanes onto the GC-bounded cache path main gained an MI350P lane in each self-hosted workflow while this branch was open, both carrying the old literal /var/tmp/rocm-e2e-uv-cache. The contract test added earlier in this branch caught them on rebase, which is the case it was written for: a new lane copied from an older one. Converted both to the derived /uv-cache. No occurrence of the old literal remains in either workflow. Signed-off-by: Tomas Saaristola --- .github/workflows/e2e-selfhosted.yml | 10 ++++++---- .github/workflows/nightly.yml | 8 +++++--- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index 4b58abc9c..3b2b9eb03 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -1073,10 +1073,12 @@ jobs: # the next job even though the container is not. export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target" export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared" - # A SEPARATE volume is mounted at exactly this path by the runner's - # deployment. Keeping the ~23GB uv cache off the work volume is - # deliberate; if you change this path, change that deployment too. - export E2E_SHARED_UV_CACHE_DIR="/var/tmp/rocm-e2e-uv-cache" + # Same mount as the runtimes it populates, so uv hardlinks into them + # instead of copying, and inside the directory the runner's uv-cache-gc + # initContainer bounds — see the e2e-gpu lane for both mechanisms and + # why a cache one level deeper keeps the links but loses the floor. + runner_root="$(dirname "$(dirname "$RUNNER_WORKSPACE")")" + export E2E_SHARED_UV_CACHE_DIR="$runner_root/uv-cache" # Pre-warm one shared runtime in place before the suite. Not an # optimization — `install sdk` bakes absolute paths into the runtime # manifest, so installing anywhere temporary breaks every later serve. diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 669edbb27..cdd40371b 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -617,9 +617,11 @@ jobs: # away, so the cargo target dir and shared caches stay warm. export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target" export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared" - # Separate volume mounted at exactly this path by the runner's - # deployment; if you change this path, change that deployment too. - export E2E_SHARED_UV_CACHE_DIR="/var/tmp/rocm-e2e-uv-cache" + # Same mount as the runtimes it populates, so uv hardlinks into them + # instead of copying, and inside the uv-cache-gc initContainer's floor + # (see e2e-selfhosted.yml e2e-gpu). + runner_root="$(dirname "$(dirname "$RUNNER_WORKSPACE")")" + export E2E_SHARED_UV_CACHE_DIR="$runner_root/uv-cache" prewarm="$RUNNER_WORKSPACE/e2e-prewarm-multi-arch-v2" export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes"