From 8a93a376374bc53bcb6e22bfbd443b2e1a721a00 Mon Sep 17 00:00:00 2001 From: Kaileshwar16 Date: Tue, 3 Nov 2026 20:47:25 +0530 Subject: [PATCH 1/4] fix(diskio): cap unpacking budget by available Linux memory --- src/diskio/mod.rs | 54 ++++++++++++--- src/diskio/test.rs | 167 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 213 insertions(+), 8 deletions(-) diff --git a/src/diskio/mod.rs b/src/diskio/mod.rs index ebcc54a847..c48bbed2c9 100644 --- a/src/diskio/mod.rs +++ b/src/diskio/mod.rs @@ -488,16 +488,38 @@ fn effective_thread_count(ram_budget: usize, thread_count: IoThreadCount) -> usi } pub(super) fn unpack_ram(io_chunk_size: usize, budget: Option) -> usize { - const RAM_ALLOWANCE_FOR_RUSTUP_AND_BUFFERS: usize = 200 * 1024 * 1024; - let minimum_ram = io_chunk_size * 2; + #[cfg(target_os = "linux")] + let available_memory = linux_available_memory(Path::new("/proc/meminfo")); + #[cfg(not(target_os = "linux"))] + let available_memory = None; + + unpack_ram_with_limits( + io_chunk_size, + budget, + effective_limits::memory_limit(), + available_memory, + ) +} - let default_max_unpack_ram = match effective_limits::memory_limit() { - Ok(effective) - if effective as usize > minimum_ram + RAM_ALLOWANCE_FOR_RUSTUP_AND_BUFFERS => - { - effective as usize - RAM_ALLOWANCE_FOR_RUSTUP_AND_BUFFERS +fn unpack_ram_with_limits( + io_chunk_size: usize, + budget: Option, + memory_limit: effective_limits::Result, + available_memory: Option, +) -> usize { + const RAM_ALLOWANCE_FOR_RUSTUP_AND_BUFFERS: usize = 200 * 1024 * 1024; + let minimum_ram = io_chunk_size.saturating_mul(2); + + let default_max_unpack_ram = match memory_limit { + Ok(effective) => { + // Physical memory and process limits don't account for other applications. + // Keep those limits, but also cap the budget by current availability. + let effective = effective.min(available_memory.unwrap_or(effective)); + usize::try_from(effective) + .unwrap_or(usize::MAX) + .saturating_sub(RAM_ALLOWANCE_FOR_RUSTUP_AND_BUFFERS) + .max(minimum_ram) } - Ok(_) => minimum_ram, Err(e) => { error!("{:#}", anyhow!(e).context("can't determine memory limit")); minimum_ram @@ -537,6 +559,22 @@ pub(super) fn unpack_ram(io_chunk_size: usize, budget: Option) -> usize { } } +#[cfg(target_os = "linux")] +fn linux_available_memory(path: &Path) -> Option { + let meminfo = std::fs::read_to_string(path).ok()?; + let available = meminfo + .lines() + .find_map(|line| line.strip_prefix("MemAvailable:"))?; + let mut fields = available.split_whitespace(); + let kib = fields.next()?.parse::().ok()?; + if fields.next()? != "kB" || fields.next().is_some() { + return None; + } + // MemAvailable includes reclaimable caches, unlike MemFree. Older kernels + // may omit it; in that case retain the effective-limits fallback. + kib.checked_mul(1024) +} + static RAM_NOTICE_SHOWN: OnceLock<()> = OnceLock::new(); /// The Threaded executor uses substantially more memory than its ram_budget diff --git a/src/diskio/test.rs b/src/diskio/test.rs index 6f876d7d14..ac5849b4dc 100644 --- a/src/diskio/test.rs +++ b/src/diskio/test.rs @@ -203,3 +203,170 @@ fn test_effective_thread_count() { 8 ); } + +#[test] +fn test_unpack_ram_uses_available_memory() { + use super::{IO_CHUNK_SIZE, effective_thread_count, unpack_ram_with_limits}; + use crate::process::IoThreadCount; + + // Issue #5173: total RAM allowed threaded unpacking despite other services + // leaving less than the low-memory threshold available. + for available_mib in [466, 508] { + let budget = unpack_ram_with_limits( + IO_CHUNK_SIZE, + None, + Ok(906 * 1024 * 1024), + Some(available_mib * 1024 * 1024), + ); + assert_eq!(effective_thread_count(budget, IoThreadCount::Default(2)), 1); + assert_eq!(budget as u64, (available_mib - 200) * 1024 * 1024); + assert!(matches!( + get_executor(budget, IoThreadCount::Default(2)).get_buffer(0), + super::FileBuffer::Immediate(_), + )); + } +} + +#[test] +fn test_unpack_ram_threshold() { + use super::{ + IO_CHUNK_SIZE, LOW_MEMORY_THRESHOLD, effective_thread_count, unpack_ram_with_limits, + }; + use crate::process::IoThreadCount::Default; + + for (expected_budget, expected_threads) in [ + (LOW_MEMORY_THRESHOLD - 1, 1), + (LOW_MEMORY_THRESHOLD, 2), + (LOW_MEMORY_THRESHOLD + 1, 2), + ] { + let budget = unpack_ram_with_limits( + IO_CHUNK_SIZE, + None, + Ok(4 * 1024 * 1024 * 1024), + Some((expected_budget + 200 * 1024 * 1024) as u64), + ); + assert_eq!(budget, expected_budget); + assert_eq!(effective_thread_count(budget, Default(2)), expected_threads); + } +} + +#[test] +fn test_unpack_ram_limits_and_fallback() { + use super::{IO_CHUNK_SIZE, unpack_ram_with_limits}; + + for limit_mib in [400, 906] { + for available in [None, Some(1024 * 1024 * 1024)] { + // None is also the non-Linux path. Availability must not raise a limit. + assert_eq!( + unpack_ram_with_limits(IO_CHUNK_SIZE, None, Ok(limit_mib * 1024 * 1024), available), + (limit_mib as usize - 200) * 1024 * 1024, + ); + } + } + for available in [None, Some(u64::MAX)] { + // Failure to determine process limits must retain the minimum fallback, + // even if the machine reports abundant available memory. + let error = std::io::Error::other("memory limit unavailable").into(); + assert_eq!( + unpack_ram_with_limits(IO_CHUNK_SIZE, None, Err(error), available), + 2 * IO_CHUNK_SIZE, + ); + } +} + +#[test] +fn test_unpack_ram_extreme_values() { + use super::{IO_CHUNK_SIZE, unpack_ram_with_limits}; + + for memory in [0, 1, 200 * 1024 * 1024, 200 * 1024 * 1024 + 1] { + for (limit, available) in [(memory, None), (u64::MAX, Some(memory))] { + assert_eq!( + unpack_ram_with_limits(IO_CHUNK_SIZE, None, Ok(limit), available), + 2 * IO_CHUNK_SIZE, + ); + } + } + assert_eq!( + unpack_ram_with_limits(IO_CHUNK_SIZE, None, Ok(u64::MAX), Some(u64::MAX)), + usize::MAX - 200 * 1024 * 1024, + ); + assert_eq!( + unpack_ram_with_limits(usize::MAX, None, Ok(0), Some(0)), + usize::MAX, + ); +} + +#[test] +fn test_unpack_ram_overrides() { + use super::{IO_CHUNK_SIZE, effective_thread_count, unpack_ram_with_limits}; + use crate::process::IoThreadCount::Default; + + for (configured, expected) in [ + ("0", 2 * IO_CHUNK_SIZE), + ("1", 2 * IO_CHUNK_SIZE), + ("67108864", 64 * 1024 * 1024), + ("1073741824", 308 * 1024 * 1024), + ("invalid", 308 * 1024 * 1024), + ] { + let vars = HashMap::from([("RUSTUP_UNPACK_RAM".into(), configured.into())]); + let tp = TestProcess::with_vars(vars); + let budget = unpack_ram_with_limits( + IO_CHUNK_SIZE, + tp.process.unpack_ram().unwrap(), + Ok(906 * 1024 * 1024), + Some(508 * 1024 * 1024), + ); + assert_eq!(budget, expected, "RUSTUP_UNPACK_RAM={configured}"); + assert_eq!(effective_thread_count(budget, Default(2)), 1); + for threads in [1, 2, 4] { + let vars = HashMap::from([("RUSTUP_IO_THREADS".into(), threads.to_string())]); + let tp = TestProcess::with_vars(vars); + assert_eq!( + effective_thread_count(budget, tp.process.io_thread_count().unwrap()), + threads, + ); + } + } +} + +#[cfg(target_os = "linux")] +#[test] +fn test_linux_available_memory() -> anyhow::Result<()> { + use super::linux_available_memory; + + let dir = test_dir()?; + let path = dir.path().join("meminfo"); + // An absent file and an unreadable file (a directory) both retain fallback. + assert_eq!(linux_available_memory(&path), None); + assert_eq!(linux_available_memory(dir.path()), None); + + for (contents, expected) in [ + ( + "MemTotal: 928456 kB\nMemAvailable: 520192 kB\n", + Some(508 * 1024 * 1024), + ), + ("MemAvailable:\t 1 kB\n", Some(1024)), + ("MemAvailable: 0 kB\n", Some(0)), + ( + "MemAvailable: 18014398509481983 kB\n", + Some(u64::MAX - 1023), + ), + ("MemAvailable: 18014398509481984 kB\n", None), + ("MemAvailable: 18446744073709551616 kB\n", None), + ("MemAvailable: -1 kB\n", None), + ("MemAvailable: invalid kB\n", None), + ("MemAvailable: 512 MB\n", None), + ("MemAvailable: 512\n", None), + ("MemAvailable: 512 kB extra\n", None), + ("MemAvailable:\n", None), + ("MemTotal: 928456 kB\nMemFree: 100 kB\n", None), + ("OtherMemAvailable: 512 kB\n", None), + ("", None), + ] { + std::fs::write(&path, contents)?; + assert_eq!(linux_available_memory(&path), expected, "{contents:?}"); + } + std::fs::write(&path, [0xff])?; + assert_eq!(linux_available_memory(&path), None); + Ok(()) +} From bbb2b5556331fe6902bb500a2ccb5cc25e24bb56 Mon Sep 17 00:00:00 2001 From: Kaileshwar16 Date: Wed, 4 Nov 2026 19:12:19 +0530 Subject: [PATCH 2/4] fix(diskio): use available memory when limit detection fails --- src/diskio/mod.rs | 48 +++++++++------------- src/diskio/test.rs | 99 ++++++++++++++++++++++++---------------------- 2 files changed, 70 insertions(+), 77 deletions(-) diff --git a/src/diskio/mod.rs b/src/diskio/mod.rs index c48bbed2c9..00a0b23e1c 100644 --- a/src/diskio/mod.rs +++ b/src/diskio/mod.rs @@ -488,43 +488,33 @@ fn effective_thread_count(ram_budget: usize, thread_count: IoThreadCount) -> usi } pub(super) fn unpack_ram(io_chunk_size: usize, budget: Option) -> usize { + const RAM_ALLOWANCE_FOR_RUSTUP_AND_BUFFERS: usize = 200 * 1024 * 1024; + let minimum_ram = io_chunk_size.saturating_mul(2); + + let memory_limit = effective_limits::memory_limit(); #[cfg(target_os = "linux")] let available_memory = linux_available_memory(Path::new("/proc/meminfo")); #[cfg(not(target_os = "linux"))] let available_memory = None; - unpack_ram_with_limits( - io_chunk_size, - budget, - effective_limits::memory_limit(), - available_memory, - ) -} - -fn unpack_ram_with_limits( - io_chunk_size: usize, - budget: Option, - memory_limit: effective_limits::Result, - available_memory: Option, -) -> usize { - const RAM_ALLOWANCE_FOR_RUSTUP_AND_BUFFERS: usize = 200 * 1024 * 1024; - let minimum_ram = io_chunk_size.saturating_mul(2); - - let default_max_unpack_ram = match memory_limit { - Ok(effective) => { - // Physical memory and process limits don't account for other applications. - // Keep those limits, but also cap the budget by current availability. - let effective = effective.min(available_memory.unwrap_or(effective)); - usize::try_from(effective) - .unwrap_or(usize::MAX) - .saturating_sub(RAM_ALLOWANCE_FOR_RUSTUP_AND_BUFFERS) - .max(minimum_ram) - } - Err(e) => { + #[cfg(test)] + let (memory_limit, available_memory) = test::MEMORY_LIMITS + .take() + .unwrap_or((memory_limit, available_memory)); + + // Physical memory and process limits don't account for other applications. + // Honor both sources when present, or use whichever is available. + let effective = match (memory_limit, available_memory) { + (Ok(effective), available) => effective.min(available.unwrap_or(effective)), + (Err(e), available) => { error!("{:#}", anyhow!(e).context("can't determine memory limit")); - minimum_ram + available.unwrap_or(0) } }; + let default_max_unpack_ram = usize::try_from(effective) + .unwrap_or(usize::MAX) + .saturating_sub(RAM_ALLOWANCE_FOR_RUSTUP_AND_BUFFERS) + .max(minimum_ram); let unpack_ram = match budget { Some(budget) => { diff --git a/src/diskio/test.rs b/src/diskio/test.rs index ac5849b4dc..9866055417 100644 --- a/src/diskio/test.rs +++ b/src/diskio/test.rs @@ -1,4 +1,4 @@ -use std::collections::HashMap; +use std::{cell::Cell, collections::HashMap}; use super::{Executor, Item, Kind, get_executor}; use crate::{process::TestProcess, test::test_dir}; @@ -206,18 +206,17 @@ fn test_effective_thread_count() { #[test] fn test_unpack_ram_uses_available_memory() { - use super::{IO_CHUNK_SIZE, effective_thread_count, unpack_ram_with_limits}; + use super::{IO_CHUNK_SIZE, effective_thread_count, unpack_ram}; use crate::process::IoThreadCount; // Issue #5173: total RAM allowed threaded unpacking despite other services // leaving less than the low-memory threshold available. for available_mib in [466, 508] { - let budget = unpack_ram_with_limits( - IO_CHUNK_SIZE, - None, + MEMORY_LIMITS.set(Some(( Ok(906 * 1024 * 1024), Some(available_mib * 1024 * 1024), - ); + ))); + let budget = unpack_ram(IO_CHUNK_SIZE, None); assert_eq!(effective_thread_count(budget, IoThreadCount::Default(2)), 1); assert_eq!(budget as u64, (available_mib - 200) * 1024 * 1024); assert!(matches!( @@ -229,9 +228,7 @@ fn test_unpack_ram_uses_available_memory() { #[test] fn test_unpack_ram_threshold() { - use super::{ - IO_CHUNK_SIZE, LOW_MEMORY_THRESHOLD, effective_thread_count, unpack_ram_with_limits, - }; + use super::{IO_CHUNK_SIZE, LOW_MEMORY_THRESHOLD, effective_thread_count, unpack_ram}; use crate::process::IoThreadCount::Default; for (expected_budget, expected_threads) in [ @@ -239,12 +236,11 @@ fn test_unpack_ram_threshold() { (LOW_MEMORY_THRESHOLD, 2), (LOW_MEMORY_THRESHOLD + 1, 2), ] { - let budget = unpack_ram_with_limits( - IO_CHUNK_SIZE, - None, + MEMORY_LIMITS.set(Some(( Ok(4 * 1024 * 1024 * 1024), Some((expected_budget + 200 * 1024 * 1024) as u64), - ); + ))); + let budget = unpack_ram(IO_CHUNK_SIZE, None); assert_eq!(budget, expected_budget); assert_eq!(effective_thread_count(budget, Default(2)), expected_threads); } @@ -252,53 +248,54 @@ fn test_unpack_ram_threshold() { #[test] fn test_unpack_ram_limits_and_fallback() { - use super::{IO_CHUNK_SIZE, unpack_ram_with_limits}; + use super::{IO_CHUNK_SIZE, unpack_ram}; for limit_mib in [400, 906] { for available in [None, Some(1024 * 1024 * 1024)] { // None is also the non-Linux path. Availability must not raise a limit. + MEMORY_LIMITS.set(Some((Ok(limit_mib * 1024 * 1024), available))); assert_eq!( - unpack_ram_with_limits(IO_CHUNK_SIZE, None, Ok(limit_mib * 1024 * 1024), available), + unpack_ram(IO_CHUNK_SIZE, None), (limit_mib as usize - 200) * 1024 * 1024, ); } } - for available in [None, Some(u64::MAX)] { - // Failure to determine process limits must retain the minimum fallback, - // even if the machine reports abundant available memory. + for (available, expected) in [ + (None, 2 * IO_CHUNK_SIZE), + (Some(0), 2 * IO_CHUNK_SIZE), + (Some(508 * 1024 * 1024), 308 * 1024 * 1024), + (Some(u64::MAX), usize::MAX - 200 * 1024 * 1024), + ] { + // Use available memory when process limits cannot be determined, and + // retain the minimum fallback when neither source is available. let error = std::io::Error::other("memory limit unavailable").into(); - assert_eq!( - unpack_ram_with_limits(IO_CHUNK_SIZE, None, Err(error), available), - 2 * IO_CHUNK_SIZE, - ); + MEMORY_LIMITS.set(Some((Err(error), available))); + assert_eq!(unpack_ram(IO_CHUNK_SIZE, None), expected); } } #[test] fn test_unpack_ram_extreme_values() { - use super::{IO_CHUNK_SIZE, unpack_ram_with_limits}; + use super::{IO_CHUNK_SIZE, unpack_ram}; for memory in [0, 1, 200 * 1024 * 1024, 200 * 1024 * 1024 + 1] { for (limit, available) in [(memory, None), (u64::MAX, Some(memory))] { - assert_eq!( - unpack_ram_with_limits(IO_CHUNK_SIZE, None, Ok(limit), available), - 2 * IO_CHUNK_SIZE, - ); + MEMORY_LIMITS.set(Some((Ok(limit), available))); + assert_eq!(unpack_ram(IO_CHUNK_SIZE, None), 2 * IO_CHUNK_SIZE); } } + MEMORY_LIMITS.set(Some((Ok(u64::MAX), Some(u64::MAX)))); assert_eq!( - unpack_ram_with_limits(IO_CHUNK_SIZE, None, Ok(u64::MAX), Some(u64::MAX)), + unpack_ram(IO_CHUNK_SIZE, None), usize::MAX - 200 * 1024 * 1024, ); - assert_eq!( - unpack_ram_with_limits(usize::MAX, None, Ok(0), Some(0)), - usize::MAX, - ); + MEMORY_LIMITS.set(Some((Ok(0), Some(0)))); + assert_eq!(unpack_ram(usize::MAX, None), usize::MAX); } #[test] fn test_unpack_ram_overrides() { - use super::{IO_CHUNK_SIZE, effective_thread_count, unpack_ram_with_limits}; + use super::{IO_CHUNK_SIZE, effective_thread_count, unpack_ram}; use crate::process::IoThreadCount::Default; for (configured, expected) in [ @@ -310,21 +307,20 @@ fn test_unpack_ram_overrides() { ] { let vars = HashMap::from([("RUSTUP_UNPACK_RAM".into(), configured.into())]); let tp = TestProcess::with_vars(vars); - let budget = unpack_ram_with_limits( - IO_CHUNK_SIZE, - tp.process.unpack_ram().unwrap(), - Ok(906 * 1024 * 1024), - Some(508 * 1024 * 1024), - ); - assert_eq!(budget, expected, "RUSTUP_UNPACK_RAM={configured}"); - assert_eq!(effective_thread_count(budget, Default(2)), 1); - for threads in [1, 2, 4] { - let vars = HashMap::from([("RUSTUP_IO_THREADS".into(), threads.to_string())]); - let tp = TestProcess::with_vars(vars); - assert_eq!( - effective_thread_count(budget, tp.process.io_thread_count().unwrap()), - threads, - ); + let error = std::io::Error::other("memory limit unavailable").into(); + for limit in [Ok(906 * 1024 * 1024), Err(error)] { + MEMORY_LIMITS.set(Some((limit, Some(508 * 1024 * 1024)))); + let budget = unpack_ram(IO_CHUNK_SIZE, tp.process.unpack_ram().unwrap()); + assert_eq!(budget, expected, "RUSTUP_UNPACK_RAM={configured}"); + assert_eq!(effective_thread_count(budget, Default(2)), 1); + for threads in [1, 2, 4] { + let vars = HashMap::from([("RUSTUP_IO_THREADS".into(), threads.to_string())]); + let tp = TestProcess::with_vars(vars); + assert_eq!( + effective_thread_count(budget, tp.process.io_thread_count().unwrap()), + threads, + ); + } } } } @@ -370,3 +366,10 @@ fn test_linux_available_memory() -> anyhow::Result<()> { assert_eq!(linux_available_memory(&path), None); Ok(()) } + +type MemoryLimits = (effective_limits::Result, Option); + +thread_local! { + // Consumed by the next unpack_ram() call, without affecting parallel tests. + pub(super) static MEMORY_LIMITS: Cell> = const { Cell::new(None) }; +} From 6021c2ccf942601878f1a839d41a9449e46b6ee3 Mon Sep 17 00:00:00 2001 From: Kaileshwar16 Date: Wed, 4 Nov 2026 19:29:19 +0530 Subject: [PATCH 3/4] refactor(diskio): clarify memory limits and remove test injection --- src/diskio/mod.rs | 26 ++++----- src/diskio/test.rs | 130 +-------------------------------------------- 2 files changed, 15 insertions(+), 141 deletions(-) diff --git a/src/diskio/mod.rs b/src/diskio/mod.rs index 00a0b23e1c..178c3e4da0 100644 --- a/src/diskio/mod.rs +++ b/src/diskio/mod.rs @@ -497,24 +497,26 @@ pub(super) fn unpack_ram(io_chunk_size: usize, budget: Option) -> usize { #[cfg(not(target_os = "linux"))] let available_memory = None; - #[cfg(test)] - let (memory_limit, available_memory) = test::MEMORY_LIMITS - .take() - .unwrap_or((memory_limit, available_memory)); - // Physical memory and process limits don't account for other applications. // Honor both sources when present, or use whichever is available. let effective = match (memory_limit, available_memory) { - (Ok(effective), available) => effective.min(available.unwrap_or(effective)), - (Err(e), available) => { + (Ok(effective), Some(available)) => Ord::min(effective, available), + (Ok(effective), None) => effective, + (Err(e), Some(available)) => { + error!("{:#}", anyhow!(e).context("can't determine memory limit")); + available + } + (Err(e), None) => { error!("{:#}", anyhow!(e).context("can't determine memory limit")); - available.unwrap_or(0) + 0 } }; - let default_max_unpack_ram = usize::try_from(effective) - .unwrap_or(usize::MAX) - .saturating_sub(RAM_ALLOWANCE_FOR_RUSTUP_AND_BUFFERS) - .max(minimum_ram); + let default_max_unpack_ram = Ord::max( + usize::try_from(effective) + .unwrap_or(usize::MAX) + .saturating_sub(RAM_ALLOWANCE_FOR_RUSTUP_AND_BUFFERS), + minimum_ram, + ); let unpack_ram = match budget { Some(budget) => { diff --git a/src/diskio/test.rs b/src/diskio/test.rs index 9866055417..9459055409 100644 --- a/src/diskio/test.rs +++ b/src/diskio/test.rs @@ -1,4 +1,4 @@ -use std::{cell::Cell, collections::HashMap}; +use std::collections::HashMap; use super::{Executor, Item, Kind, get_executor}; use crate::{process::TestProcess, test::test_dir}; @@ -204,127 +204,6 @@ fn test_effective_thread_count() { ); } -#[test] -fn test_unpack_ram_uses_available_memory() { - use super::{IO_CHUNK_SIZE, effective_thread_count, unpack_ram}; - use crate::process::IoThreadCount; - - // Issue #5173: total RAM allowed threaded unpacking despite other services - // leaving less than the low-memory threshold available. - for available_mib in [466, 508] { - MEMORY_LIMITS.set(Some(( - Ok(906 * 1024 * 1024), - Some(available_mib * 1024 * 1024), - ))); - let budget = unpack_ram(IO_CHUNK_SIZE, None); - assert_eq!(effective_thread_count(budget, IoThreadCount::Default(2)), 1); - assert_eq!(budget as u64, (available_mib - 200) * 1024 * 1024); - assert!(matches!( - get_executor(budget, IoThreadCount::Default(2)).get_buffer(0), - super::FileBuffer::Immediate(_), - )); - } -} - -#[test] -fn test_unpack_ram_threshold() { - use super::{IO_CHUNK_SIZE, LOW_MEMORY_THRESHOLD, effective_thread_count, unpack_ram}; - use crate::process::IoThreadCount::Default; - - for (expected_budget, expected_threads) in [ - (LOW_MEMORY_THRESHOLD - 1, 1), - (LOW_MEMORY_THRESHOLD, 2), - (LOW_MEMORY_THRESHOLD + 1, 2), - ] { - MEMORY_LIMITS.set(Some(( - Ok(4 * 1024 * 1024 * 1024), - Some((expected_budget + 200 * 1024 * 1024) as u64), - ))); - let budget = unpack_ram(IO_CHUNK_SIZE, None); - assert_eq!(budget, expected_budget); - assert_eq!(effective_thread_count(budget, Default(2)), expected_threads); - } -} - -#[test] -fn test_unpack_ram_limits_and_fallback() { - use super::{IO_CHUNK_SIZE, unpack_ram}; - - for limit_mib in [400, 906] { - for available in [None, Some(1024 * 1024 * 1024)] { - // None is also the non-Linux path. Availability must not raise a limit. - MEMORY_LIMITS.set(Some((Ok(limit_mib * 1024 * 1024), available))); - assert_eq!( - unpack_ram(IO_CHUNK_SIZE, None), - (limit_mib as usize - 200) * 1024 * 1024, - ); - } - } - for (available, expected) in [ - (None, 2 * IO_CHUNK_SIZE), - (Some(0), 2 * IO_CHUNK_SIZE), - (Some(508 * 1024 * 1024), 308 * 1024 * 1024), - (Some(u64::MAX), usize::MAX - 200 * 1024 * 1024), - ] { - // Use available memory when process limits cannot be determined, and - // retain the minimum fallback when neither source is available. - let error = std::io::Error::other("memory limit unavailable").into(); - MEMORY_LIMITS.set(Some((Err(error), available))); - assert_eq!(unpack_ram(IO_CHUNK_SIZE, None), expected); - } -} - -#[test] -fn test_unpack_ram_extreme_values() { - use super::{IO_CHUNK_SIZE, unpack_ram}; - - for memory in [0, 1, 200 * 1024 * 1024, 200 * 1024 * 1024 + 1] { - for (limit, available) in [(memory, None), (u64::MAX, Some(memory))] { - MEMORY_LIMITS.set(Some((Ok(limit), available))); - assert_eq!(unpack_ram(IO_CHUNK_SIZE, None), 2 * IO_CHUNK_SIZE); - } - } - MEMORY_LIMITS.set(Some((Ok(u64::MAX), Some(u64::MAX)))); - assert_eq!( - unpack_ram(IO_CHUNK_SIZE, None), - usize::MAX - 200 * 1024 * 1024, - ); - MEMORY_LIMITS.set(Some((Ok(0), Some(0)))); - assert_eq!(unpack_ram(usize::MAX, None), usize::MAX); -} - -#[test] -fn test_unpack_ram_overrides() { - use super::{IO_CHUNK_SIZE, effective_thread_count, unpack_ram}; - use crate::process::IoThreadCount::Default; - - for (configured, expected) in [ - ("0", 2 * IO_CHUNK_SIZE), - ("1", 2 * IO_CHUNK_SIZE), - ("67108864", 64 * 1024 * 1024), - ("1073741824", 308 * 1024 * 1024), - ("invalid", 308 * 1024 * 1024), - ] { - let vars = HashMap::from([("RUSTUP_UNPACK_RAM".into(), configured.into())]); - let tp = TestProcess::with_vars(vars); - let error = std::io::Error::other("memory limit unavailable").into(); - for limit in [Ok(906 * 1024 * 1024), Err(error)] { - MEMORY_LIMITS.set(Some((limit, Some(508 * 1024 * 1024)))); - let budget = unpack_ram(IO_CHUNK_SIZE, tp.process.unpack_ram().unwrap()); - assert_eq!(budget, expected, "RUSTUP_UNPACK_RAM={configured}"); - assert_eq!(effective_thread_count(budget, Default(2)), 1); - for threads in [1, 2, 4] { - let vars = HashMap::from([("RUSTUP_IO_THREADS".into(), threads.to_string())]); - let tp = TestProcess::with_vars(vars); - assert_eq!( - effective_thread_count(budget, tp.process.io_thread_count().unwrap()), - threads, - ); - } - } - } -} - #[cfg(target_os = "linux")] #[test] fn test_linux_available_memory() -> anyhow::Result<()> { @@ -366,10 +245,3 @@ fn test_linux_available_memory() -> anyhow::Result<()> { assert_eq!(linux_available_memory(&path), None); Ok(()) } - -type MemoryLimits = (effective_limits::Result, Option); - -thread_local! { - // Consumed by the next unpack_ram() call, without affecting parallel tests. - pub(super) static MEMORY_LIMITS: Cell> = const { Cell::new(None) }; -} From cbecb2b662ff5015d5441ca688d13e0d5b468356 Mon Sep 17 00:00:00 2001 From: Kaileshwar16 Date: Thu, 5 Nov 2026 10:19:31 +0530 Subject: [PATCH 4/4] refactor(diskio): use cfg_select for available memory --- src/diskio/mod.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/diskio/mod.rs b/src/diskio/mod.rs index 178c3e4da0..aabb552472 100644 --- a/src/diskio/mod.rs +++ b/src/diskio/mod.rs @@ -492,10 +492,10 @@ pub(super) fn unpack_ram(io_chunk_size: usize, budget: Option) -> usize { let minimum_ram = io_chunk_size.saturating_mul(2); let memory_limit = effective_limits::memory_limit(); - #[cfg(target_os = "linux")] - let available_memory = linux_available_memory(Path::new("/proc/meminfo")); - #[cfg(not(target_os = "linux"))] - let available_memory = None; + let available_memory = cfg_select! { + target_os = "linux" => linux_available_memory(Path::new("/proc/meminfo")), + _ => None, + }; // Physical memory and process limits don't account for other applications. // Honor both sources when present, or use whichever is available.