diff --git a/src/diskio/mod.rs b/src/diskio/mod.rs index ebcc54a847..aabb552472 100644 --- a/src/diskio/mod.rs +++ b/src/diskio/mod.rs @@ -489,20 +489,34 @@ fn effective_thread_count(ram_budget: usize, thread_count: IoThreadCount) -> usi pub(super) fn unpack_ram(io_chunk_size: usize, budget: Option) -> usize { const RAM_ALLOWANCE_FOR_RUSTUP_AND_BUFFERS: usize = 200 * 1024 * 1024; - let minimum_ram = io_chunk_size * 2; + let minimum_ram = io_chunk_size.saturating_mul(2); - let default_max_unpack_ram = match effective_limits::memory_limit() { - Ok(effective) - if effective as usize > minimum_ram + RAM_ALLOWANCE_FOR_RUSTUP_AND_BUFFERS => - { - effective as usize - RAM_ALLOWANCE_FOR_RUSTUP_AND_BUFFERS + let memory_limit = effective_limits::memory_limit(); + let available_memory = cfg_select! { + target_os = "linux" => linux_available_memory(Path::new("/proc/meminfo")), + _ => None, + }; + + // Physical memory and process limits don't account for other applications. + // Honor both sources when present, or use whichever is available. + let effective = match (memory_limit, available_memory) { + (Ok(effective), Some(available)) => Ord::min(effective, available), + (Ok(effective), None) => effective, + (Err(e), Some(available)) => { + error!("{:#}", anyhow!(e).context("can't determine memory limit")); + available } - Ok(_) => minimum_ram, - Err(e) => { + (Err(e), None) => { error!("{:#}", anyhow!(e).context("can't determine memory limit")); - minimum_ram + 0 } }; + let default_max_unpack_ram = Ord::max( + usize::try_from(effective) + .unwrap_or(usize::MAX) + .saturating_sub(RAM_ALLOWANCE_FOR_RUSTUP_AND_BUFFERS), + minimum_ram, + ); let unpack_ram = match budget { Some(budget) => { @@ -537,6 +551,22 @@ pub(super) fn unpack_ram(io_chunk_size: usize, budget: Option) -> usize { } } +#[cfg(target_os = "linux")] +fn linux_available_memory(path: &Path) -> Option { + let meminfo = std::fs::read_to_string(path).ok()?; + let available = meminfo + .lines() + .find_map(|line| line.strip_prefix("MemAvailable:"))?; + let mut fields = available.split_whitespace(); + let kib = fields.next()?.parse::().ok()?; + if fields.next()? != "kB" || fields.next().is_some() { + return None; + } + // MemAvailable includes reclaimable caches, unlike MemFree. Older kernels + // may omit it; in that case retain the effective-limits fallback. + kib.checked_mul(1024) +} + static RAM_NOTICE_SHOWN: OnceLock<()> = OnceLock::new(); /// The Threaded executor uses substantially more memory than its ram_budget diff --git a/src/diskio/test.rs b/src/diskio/test.rs index 6f876d7d14..9459055409 100644 --- a/src/diskio/test.rs +++ b/src/diskio/test.rs @@ -203,3 +203,45 @@ fn test_effective_thread_count() { 8 ); } + +#[cfg(target_os = "linux")] +#[test] +fn test_linux_available_memory() -> anyhow::Result<()> { + use super::linux_available_memory; + + let dir = test_dir()?; + let path = dir.path().join("meminfo"); + // An absent file and an unreadable file (a directory) both retain fallback. + assert_eq!(linux_available_memory(&path), None); + assert_eq!(linux_available_memory(dir.path()), None); + + for (contents, expected) in [ + ( + "MemTotal: 928456 kB\nMemAvailable: 520192 kB\n", + Some(508 * 1024 * 1024), + ), + ("MemAvailable:\t 1 kB\n", Some(1024)), + ("MemAvailable: 0 kB\n", Some(0)), + ( + "MemAvailable: 18014398509481983 kB\n", + Some(u64::MAX - 1023), + ), + ("MemAvailable: 18014398509481984 kB\n", None), + ("MemAvailable: 18446744073709551616 kB\n", None), + ("MemAvailable: -1 kB\n", None), + ("MemAvailable: invalid kB\n", None), + ("MemAvailable: 512 MB\n", None), + ("MemAvailable: 512\n", None), + ("MemAvailable: 512 kB extra\n", None), + ("MemAvailable:\n", None), + ("MemTotal: 928456 kB\nMemFree: 100 kB\n", None), + ("OtherMemAvailable: 512 kB\n", None), + ("", None), + ] { + std::fs::write(&path, contents)?; + assert_eq!(linux_available_memory(&path), expected, "{contents:?}"); + } + std::fs::write(&path, [0xff])?; + assert_eq!(linux_available_memory(&path), None); + Ok(()) +}