fix(diskio): cap unpacking budget by available Linux memory

src/diskio/mod.rs

@@ -489,20 +489,34 @@ fn effective_thread_count(ram_budget: usize, thread_count: IoThreadCount) -> usi
 
 pub(super) fn unpack_ram(io_chunk_size: usize, budget: Option<usize>) -> usize {
     const RAM_ALLOWANCE_FOR_RUSTUP_AND_BUFFERS: usize = 200 * 1024 * 1024;
-    let minimum_ram = io_chunk_size * 2;
+    let minimum_ram = io_chunk_size.saturating_mul(2);
 
-    let default_max_unpack_ram = match effective_limits::memory_limit() {
-        Ok(effective)
-            if effective as usize > minimum_ram + RAM_ALLOWANCE_FOR_RUSTUP_AND_BUFFERS =>
-        {
-            effective as usize - RAM_ALLOWANCE_FOR_RUSTUP_AND_BUFFERS
+    let memory_limit = effective_limits::memory_limit();
+    let available_memory = cfg_select! {
+        target_os = "linux" => linux_available_memory(Path::new("/proc/meminfo")),
+        _ => None,
+    };
+
+    // Physical memory and process limits don't account for other applications.
+    // Honor both sources when present, or use whichever is available.
+    let effective = match (memory_limit, available_memory) {
+        (Ok(effective), Some(available)) => Ord::min(effective, available),
+        (Ok(effective), None) => effective,
+        (Err(e), Some(available)) => {
+            error!("{:#}", anyhow!(e).context("can't determine memory limit"));
+            available
         }
-        Ok(_) => minimum_ram,
-        Err(e) => {
+        (Err(e), None) => {
             error!("{:#}", anyhow!(e).context("can't determine memory limit"));
-            minimum_ram
+            0
         }
     };
+    let default_max_unpack_ram = Ord::max(
+        usize::try_from(effective)
+            .unwrap_or(usize::MAX)
+            .saturating_sub(RAM_ALLOWANCE_FOR_RUSTUP_AND_BUFFERS),
+        minimum_ram,
+    );
 
     let unpack_ram = match budget {
         Some(budget) => {
@@ -537,6 +551,22 @@ pub(super) fn unpack_ram(io_chunk_size: usize, budget: Option<usize>) -> usize {
     }
 }
 
+#[cfg(target_os = "linux")]
+fn linux_available_memory(path: &Path) -> Option<u64> {
+    let meminfo = std::fs::read_to_string(path).ok()?;
+    let available = meminfo
+        .lines()
+        .find_map(|line| line.strip_prefix("MemAvailable:"))?;
+    let mut fields = available.split_whitespace();
+    let kib = fields.next()?.parse::<u64>().ok()?;
+    if fields.next()? != "kB" || fields.next().is_some() {
+        return None;
+    }
+    // MemAvailable includes reclaimable caches, unlike MemFree. Older kernels
+    // may omit it; in that case retain the effective-limits fallback.
+    kib.checked_mul(1024)
+}
+
 static RAM_NOTICE_SHOWN: OnceLock<()> = OnceLock::new();
 
 /// The Threaded executor uses substantially more memory than its ram_budget

src/diskio/test.rs

@@ -203,3 +203,45 @@ fn test_effective_thread_count() {
         8
     );
 }
+
+#[cfg(target_os = "linux")]
+#[test]
+fn test_linux_available_memory() -> anyhow::Result<()> {
+    use super::linux_available_memory;
+
+    let dir = test_dir()?;
+    let path = dir.path().join("meminfo");
+    // An absent file and an unreadable file (a directory) both retain fallback.
+    assert_eq!(linux_available_memory(&path), None);
+    assert_eq!(linux_available_memory(dir.path()), None);
+
+    for (contents, expected) in [
+        (
+            "MemTotal: 928456 kB\nMemAvailable: 520192 kB\n",
+            Some(508 * 1024 * 1024),
+        ),
+        ("MemAvailable:\t  1 kB\n", Some(1024)),
+        ("MemAvailable: 0 kB\n", Some(0)),
+        (
+            "MemAvailable: 18014398509481983 kB\n",
+            Some(u64::MAX - 1023),
+        ),
+        ("MemAvailable: 18014398509481984 kB\n", None),
+        ("MemAvailable: 18446744073709551616 kB\n", None),
+        ("MemAvailable: -1 kB\n", None),
+        ("MemAvailable: invalid kB\n", None),
+        ("MemAvailable: 512 MB\n", None),
+        ("MemAvailable: 512\n", None),
+        ("MemAvailable: 512 kB extra\n", None),
+        ("MemAvailable:\n", None),
+        ("MemTotal: 928456 kB\nMemFree: 100 kB\n", None),
+        ("OtherMemAvailable: 512 kB\n", None),
+        ("", None),
+    ] {
+        std::fs::write(&path, contents)?;
+        assert_eq!(linux_available_memory(&path), expected, "{contents:?}");
+    }
+    std::fs::write(&path, [0xff])?;
+    assert_eq!(linux_available_memory(&path), None);
+    Ok(())
+}