ci(e2e): colocate the uv cache so uv hardlinks instead of copying

.github/workflows/e2e-selfhosted.yml

@@ -257,11 +257,39 @@ jobs:
           # scenario. Persistent path; service state stays isolated per scenario.
           export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared"
           # Share uv's wheel download/build cache so `rocm install sdk` is a warm
-          # ~34s per scenario instead of a cold ~160s (measured on MI300X). Kept
-          # OFF the RUNNER_WORKSPACE Longhorn PVC (near-full) and on the roomy `/`
-          # overlay; ~23GB, one cold fill per pod-life. Not git-cleaned (outside
-          # the checkout) and outside the reclaim step's /tmp/rocm-e2e-* glob.
-          export E2E_SHARED_UV_CACHE_DIR="/var/tmp/rocm-e2e-uv-cache"
+          # ~34s per scenario instead of a cold ~160s (measured on MI300X).
+          #
+          # On RUNNER_WORKSPACE, next to the runtimes it populates, because uv can
+          # only hardlink out of this cache into a managed environment when the two
+          # are reachable without crossing a mount point. Anywhere else and it
+          # silently falls back to copying every file, so each runtime carries its
+          # own full copy of the ROCm SDK and torch stack. uv exits 0 either way and
+          # two of the three install paths discard its warning, so the fallback is
+          # invisible unless you compare inodes.
+          #
+          # ~23GB per ROCm generation, and uv never evicts, so this shares the work
+          # volume's budget with e2e-target/e2e-shared/e2e-prewarm rather than
+          # having a quota of its own. What keeps it from starving them — and from
+          # starving `.runner`, whose credentials live on the same volume and need a
+          # repo-Administration token to re-register — is the runner pod's
+          # `uv-cache-gc` initContainer. On the way up it sweeps abandoned `.tmp*`
+          # extractions, then deletes `archive-v0` outright if FREE space on the
+          # volume is under 60GiB. It is a free-space floor, not a size cap: the
+          # floor covers two full ~25GB fills, because a ROCm generation transition
+          # installs the new runtime beside the old one and both are resident until
+          # the next pod start. The work volume is ~295G, so the floor has room to
+          # mean something.
+          #
+          # That GC bounds ONE directory: the `uv-cache` subPath of the work PVC,
+          # which surfaces in this container as <runner-root>/uv-cache. Pointing the
+          # cache anywhere else on the volume keeps the hardlinks but escapes the
+          # floor. The 49G/49G strand that motivated all of this was a *different*
+          # volume — the dedicated 50Gi cache PVC, since deleted — where no floor
+          # existed at all; escaping this one puts the same failure back on a volume
+          # that also holds the runner credentials.
+          # $RUNNER_WORKSPACE is <runner-root>/_work/<repo>, so derive the root.
+          runner_root="$(dirname "$(dirname "$RUNNER_WORKSPACE")")"
+          export E2E_SHARED_UV_CACHE_DIR="$runner_root/uv-cache"
           # Share ONE installed managed runtime across the serve/chat scenarios so
           # `rocm install sdk` runs once per runner, not once per scenario (the
           # per-scenario install count — each a multi-GiB TheRock SDK whose probe
@@ -916,10 +944,12 @@ jobs:
           # still warm on the next job even though the container is not.
           export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target"
           export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared"
-          # A SEPARATE PVC is mounted at exactly this path by the runner's cluster
-          # overlay. Keeping the ~23GB uv cache off the work PVC is deliberate; if
-          # you change this path, change the overlay that mounts it too.
-          export E2E_SHARED_UV_CACHE_DIR="/var/tmp/rocm-e2e-uv-cache"
+          # Same mount as the runtimes it populates, so uv hardlinks into them
+          # instead of copying, and inside the directory the runner's uv-cache-gc
+          # initContainer bounds — see the e2e-gpu lane for why the distinction is
+          # invisible in the logs and why the GC's path is load-bearing.
+          runner_root="$(dirname "$(dirname "$RUNNER_WORKSPACE")")"
+          export E2E_SHARED_UV_CACHE_DIR="$runner_root/uv-cache"
           # Pre-warm one shared runtime in place before the suite. Not an
           # optimization — `install sdk` bakes absolute paths into the runtime
           # manifest, so installing anywhere temporary breaks every later serve.
@@ -1043,10 +1073,12 @@ jobs:
           # the next job even though the container is not.
           export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target"
           export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared"
-          # A SEPARATE volume is mounted at exactly this path by the runner's
-          # deployment. Keeping the ~23GB uv cache off the work volume is
-          # deliberate; if you change this path, change that deployment too.
-          export E2E_SHARED_UV_CACHE_DIR="/var/tmp/rocm-e2e-uv-cache"
+          # Same mount as the runtimes it populates, so uv hardlinks into them
+          # instead of copying, and inside the directory the runner's uv-cache-gc
+          # initContainer bounds — see the e2e-gpu lane for both mechanisms and
+          # why a cache one level deeper keeps the links but loses the floor.
+          runner_root="$(dirname "$(dirname "$RUNNER_WORKSPACE")")"
+          export E2E_SHARED_UV_CACHE_DIR="$runner_root/uv-cache"
           # Pre-warm one shared runtime in place before the suite. Not an
           # optimization — `install sdk` bakes absolute paths into the runtime
           # manifest, so installing anywhere temporary breaks every later serve.

.github/workflows/nightly.yml

@@ -332,10 +332,11 @@ jobs:
         run: gh release delete "${{ needs.nightly.outputs.staging_tag }}" --yes --cleanup-tag 2>/dev/null || true
 
   # Full E2E on app-dev MI300X INCLUDING the expensive `@nightly` scenarios (the
-  # large-model serve), which the per-PR ci.yml `e2e-gpu` job skips to stay fast.
+  # large-model serve), which the per-PR e2e-selfhosted.yml `e2e-gpu` job skips
+  # to stay fast.
   # Runs unconditionally on the schedule/dispatch (independent of the nightly
   # BUILD skip above — a regression run is worth doing even with no new commits).
-  # Non-blocking, mirrors ci.yml's e2e-gpu job but sets E2E_INCLUDE_NIGHTLY.
+  # Non-blocking, mirrors e2e-selfhosted.yml's e2e-gpu job but sets E2E_INCLUDE_NIGHTLY.
   e2e-gpu-nightly:
     name: E2E tests (MI300X, incl. nightly-only)
     timeout-minutes: 90
@@ -353,7 +354,7 @@ jobs:
 
       # Reclaim the GPU before running: kill only e2e leftovers scoped to
       # /tmp/rocm-e2e-* and the e2e-target/e2e-shared trees — never the runner or
-      # any /workload manual-testing processes (see ci.yml e2e-gpu for the same).
+      # any /workload manual-testing processes (see e2e-selfhosted.yml e2e-gpu for the same).
       - name: Reclaim GPU from stray E2E processes
         run: |
           pkill -f '/tmp/rocm-e2e.*llama-server' 2>/dev/null || true
@@ -414,11 +415,15 @@ jobs:
         run: |
           export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target"
           export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared"
-          # Share uv's wheel cache off the near-full PVC, on the roomy `/` overlay
-          # (see ci.yml e2e-gpu): warm `rocm install sdk` ~34s vs cold ~160s.
-          export E2E_SHARED_UV_CACHE_DIR="/var/tmp/rocm-e2e-uv-cache"
+          # Share uv's wheel cache on the same mount as the runtimes it populates,
+          # so uv hardlinks into them instead of copying (see e2e-selfhosted.yml
+          # e2e-gpu): warm `rocm install sdk` ~34s vs cold ~160s. It must also be
+          # the directory the runner's uv-cache-gc initContainer bounds, or the
+          # cache grows without a floor on the volume holding `.runner`.
+          runner_root="$(dirname "$(dirname "$RUNNER_WORKSPACE")")"
+          export E2E_SHARED_UV_CACHE_DIR="$runner_root/uv-cache"
           # Share ONE installed managed runtime across serve/chat scenarios so
-          # install sdk runs once per runner, not per scenario (see ci.yml e2e-gpu).
+          # install sdk runs once per runner, not per scenario (see e2e-selfhosted.yml e2e-gpu).
           # The shared dir IS the pre-warm's own data/runtimes; NEVER move it — a
           # post-install mv invalidates the absolute paths install sdk bakes into
           # the runtime manifest and every serve fails instantly (run 29320025393).
@@ -456,8 +461,8 @@ jobs:
           path: tests/e2e-cucumber/results/
 
   # rad3 R9700 counterpart to e2e-gpu-nightly. Mirrors the per-PR e2e-gpu-rad3
-  # lane in e2e-selfhosted.yml (scale-to-zero runner, r9700 label, own uv-cache
-  # PVC, GPU preflight), with @nightly scenarios enabled. Keep the two in step:
+  # lane in e2e-selfhosted.yml (scale-to-zero runner, r9700 label, shared uv
+  # cache, GPU preflight), with @nightly scenarios enabled. Keep the two in step:
   # the report crate asserts the nightly lanes publish the same platforms as the
   # per-PR lanes.
   e2e-gpu-nightly-rad3:
@@ -521,9 +526,11 @@ jobs:
           # being scaled away, so the cargo target dir and shared caches stay warm.
           export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target"
           export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared"
-          # Separate PVC mounted at exactly this path by the runner's cluster
-          # overlay; if you change this path, change the overlay that mounts it too.
-          export E2E_SHARED_UV_CACHE_DIR="/var/tmp/rocm-e2e-uv-cache"
+          # Same mount as the runtimes it populates, so uv hardlinks into them
+          # instead of copying, and inside the uv-cache-gc initContainer's floor
+          # (see e2e-selfhosted.yml e2e-gpu).
+          runner_root="$(dirname "$(dirname "$RUNNER_WORKSPACE")")"
+          export E2E_SHARED_UV_CACHE_DIR="$runner_root/uv-cache"
           prewarm="$RUNNER_WORKSPACE/e2e-prewarm-multi-arch-v2"
           export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes"
 
@@ -610,9 +617,11 @@ jobs:
           # away, so the cargo target dir and shared caches stay warm.
           export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target"
           export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared"
-          # Separate volume mounted at exactly this path by the runner's
-          # deployment; if you change this path, change that deployment too.
-          export E2E_SHARED_UV_CACHE_DIR="/var/tmp/rocm-e2e-uv-cache"
+          # Same mount as the runtimes it populates, so uv hardlinks into them
+          # instead of copying, and inside the uv-cache-gc initContainer's floor
+          # (see e2e-selfhosted.yml e2e-gpu).
+          runner_root="$(dirname "$(dirname "$RUNNER_WORKSPACE")")"
+          export E2E_SHARED_UV_CACHE_DIR="$runner_root/uv-cache"
           prewarm="$RUNNER_WORKSPACE/e2e-prewarm-multi-arch-v2"
           export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes"
 
@@ -640,7 +649,8 @@ jobs:
 
   # Strix Halo counterpart to e2e-gpu-nightly. The same @nightly large-model
   # scenario serves the platform-specific Lemonade GGUF on this host. Keep the
-  # runner's storage and runtime setup aligned with ci.yml's proven Strix Ubuntu job.
+  # runner's storage and runtime setup aligned with e2e-selfhosted.yml's proven
+  # Strix Ubuntu job.
   e2e-gpu-nightly-strix:
     name: E2E tests (Strix Halo, incl. nightly-only)
     timeout-minutes: 90
@@ -733,8 +743,9 @@ jobs:
           path: tests/e2e-cucumber/results/
 
   # Windows counterpart to e2e-gpu-nightly-strix. Keep the PowerShell-native
-  # setup aligned with ci.yml's proven Strix Windows job while opting into the
-  # same platform-adaptive @nightly large-model scenario as the Linux runner.
+  # setup aligned with e2e-selfhosted.yml's proven Strix Windows job while
+  # opting into the same platform-adaptive @nightly large-model scenario as the
+  # Linux runner.
   e2e-gpu-nightly-strix-windows:
     name: E2E tests (Strix Halo, Windows, incl. nightly-only)
     timeout-minutes: 90

MANIFEST.md

@@ -669,13 +669,15 @@ The binary is cached in the rocm-cli managed data directory and reused for
 subsequent operations. The version may be pinned via `ROCM_CLI_UV_VERSION`.
 
 `uv`'s own content-addressed package cache is also kept in the managed data
-directory (at `<data-dir>/uv-cache`), so that it shares a filesystem with the
-environments `uv` populates and packages can be hardlinked into them instead of
-copied. This cache holds every wheel `uv` downloads — the ROCm SDK and the
-torch stack included — so it is typically the largest directory rocm-cli
-manages, on the order of several GB per SDK version installed. It is removed by
-`rocm uninstall` unless `--keep-data` is passed, and its location can be
-overridden with `ROCM_CLI_UV_CACHE_DIR`.
+directory (at `<data-dir>/uv-cache`), so that it is reachable from the
+environments `uv` populates without crossing a mount point and packages can be
+hardlinked into them instead of copied. Note it is the mount, not the
+filesystem: a bind mount or `subPath` volume is enough to make Linux refuse the
+hardlink, and `uv` then silently copies. This cache holds every wheel `uv`
+downloads — the ROCm SDK and the torch stack included — so it is typically the
+largest directory rocm-cli manages, on the order of several GB per SDK version
+installed. It is removed by `rocm uninstall` unless `--keep-data` is passed, and
+its location can be overridden with `ROCM_CLI_UV_CACHE_DIR`.
 
 ### Lemonade Embeddable Runtime
 

apps/rocm/src/therock.rs

@@ -3541,10 +3541,12 @@ fn preflight_tarball_space(
         disk_space::with_margin(download_bytes),
     )?;
 
-    // When the cache and the install root share a filesystem, the archive and
-    // the extracted tree must both fit at the same time.
+    // When the cache and the install root share a mount, the archive and the
+    // extracted tree must both fit at the same time. Conservative in the other
+    // direction: two mounts backed by one filesystem also share a pool, and this
+    // treats them as separate, so the estimate can come in under the true need.
     let mut extract_estimate = disk_space::estimated_extracted_size(download_bytes);
-    if disk_space::on_same_filesystem(cache_path, install_root) == Some(true) {
+    if disk_space::on_same_mount(cache_path, install_root) == Some(true) {
         extract_estimate = extract_estimate.saturating_add(download_bytes);
     }
     Ok(disk_space::warn_if_low_space(
@@ -7318,15 +7320,17 @@ mod tests {
         let install_root = managed_runtime_root(&paths, "wheel", runtime_key);
         assert!(install_root.starts_with(&paths.data_dir));
         // Without --prefix the generated runtime folder is itself under the data dir, so
-        // the uv cache shares a filesystem with the environment it populates.
+        // the uv cache is reachable from the environment it populates without crossing a
+        // mount point.
         assert!(managed_uv_cache_dir(&paths.data_dir).starts_with(&paths.data_dir));
     }
 
     #[test]
     fn uv_cache_does_not_follow_a_prefix_install_root() {
         // Documents a known gap rather than an intended behavior: `--prefix` relocates
-        // install_root only, while the uv cache stays keyed off the data dir. When the two
-        // land on different filesystems uv falls back to copying. Tracked separately; see
+        // install_root only, while the uv cache stays keyed off the data dir. When reaching
+        // one from the other crosses a mount point uv falls back to copying — it is the
+        // mount, not the filesystem, so a bind mount is enough. Tracked separately; see
         // the `--prefix` non-goal on the PR that introduced the colocation.
         let (_root, paths) = test_paths("prefix-uv-cache");
         let prefix_root = PathBuf::from("/mnt/elsewhere/envs/my-env");

crates/rocm-core/src/disk_space.rs

@@ -241,10 +241,17 @@ pub fn available_space_for_path(path: &Path) -> Option<u64> {
     mount_for_path(path).map(|(_, available)| available)
 }
 
-/// Whether two paths live on the same filesystem.
+/// Whether two paths live under the same mount point.
 ///
-/// `None` when either path's filesystem cannot be determined.
-pub fn on_same_filesystem(left: &Path, right: &Path) -> Option<bool> {
+/// Named for the mount deliberately: two paths can share a filesystem and still
+/// sit on different mounts (a bind mount or a `subPath` volume is enough), and
+/// the mount is what the kernel enforces — `link(2)` returns `EXDEV` across two
+/// mounts of one filesystem. Callers reasoning about hardlinks want this. A
+/// caller reasoning about shared free space does not: those two mounts draw on
+/// one pool, and this returns `false` for them.
+///
+/// `None` when either path's mount cannot be determined.
+pub fn on_same_mount(left: &Path, right: &Path) -> Option<bool> {
     let left_mount = mount_for_path(left)?.0;
     let right_mount = mount_for_path(right)?.0;
     Some(left_mount == right_mount)

crates/rocm-core/src/lib.rs

@@ -42,7 +42,7 @@ pub use diagnose::{
 pub use disk_space::{
     SpaceCheck, available_space_for_path, check_space_for_path, ensure_space_for,
     estimated_extracted_size, format_bytes, insufficient_space_message, map_write_error,
-    mount_for_path, on_same_filesystem, warn_if_low_space, with_margin,
+    mount_for_path, on_same_mount, warn_if_low_space, with_margin,
 };
 use examine::extract_rocm_version;
 pub use examine::{

crates/rocm-core/src/runtime.rs

@@ -300,8 +300,12 @@ pub fn managed_pip_cache_dir(root: &Path) -> PathBuf {
     normalize_runtime_path_for_host(root).join("pip-cache")
 }
 
-/// `uv`'s content-addressed cache, kept under the managed root so it shares a filesystem
-/// with the environments `uv` populates and hardlinking keeps working (see issue #160).
+/// `uv`'s content-addressed cache, kept under the managed root (see issue #160).
+///
+/// Colocating it keeps the cache reachable from the environments `uv` populates without
+/// crossing a mount point, which is what lets `uv` hardlink into them instead of copying.
+/// It is the mount, not the filesystem: a bind mount or `subPath` volume is enough to make
+/// Linux refuse the hardlink and send `uv` back to copying.
 pub fn managed_uv_cache_dir(root: &Path) -> PathBuf {
     normalize_runtime_path_for_host(root).join("uv-cache")
 }

crates/rocm-core/src/uv.rs

@@ -147,9 +147,11 @@ fn meaningful_cache_dir(value: Option<&OsStr>) -> Option<PathBuf> {
 /// populates.
 ///
 /// Without a cache inside the managed root, `uv` caches under `$HOME/.cache/uv`; when
-/// that is on a different filesystem from the data directory, `uv` cannot hardlink and
+/// reaching that from the data directory crosses a mount point, `uv` cannot hardlink and
 /// silently copies every file, so each environment carries a full duplicate of the SDK
-/// and torch stack.
+/// and torch stack. Note it is the mount, not the filesystem: Linux refuses a hardlink
+/// across two mounts even when both resolve to the same underlying filesystem, so a bind
+/// mount or a `subPath` volume is enough to trigger the copy fallback.
 ///
 /// Note this colocates with the *data directory*, not with a `--prefix` install root; see
 /// the `--prefix` caveat in `docs/manual-testing.md`.