From 62496f4fcf8309a70c69c325dc71dc58bb4b1f8b Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues <16357187+eduardomourar@users.noreply.github.com> Date: Sat, 1 Aug 2026 23:44:56 +0100 Subject: [PATCH] fix(wasi): sort ordering check and external-merge separator sort's ordering check runs single-threaded on WASI, which has no thread support, reading every chunk up front instead of streaming from a background reader thread. Its external-merge path also inserts a missing separator between concatenated files so lines don't merge across file boundaries. --- src/uu/sort/src/check.rs | 129 ++++++++++++++++++++++++++++--- src/uu/sort/src/ext_sort/wasi.rs | 7 ++ tests/by-util/test_sort.rs | 86 ++++++++++++++++++++- 3 files changed, 211 insertions(+), 11 deletions(-) diff --git a/src/uu/sort/src/check.rs b/src/uu/sort/src/check.rs index a826bc75507..f161426bd79 100644 --- a/src/uu/sort/src/check.rs +++ b/src/uu/sort/src/check.rs @@ -11,9 +11,9 @@ use crate::{ compare_by, open, }; use itertools::Itertools; +use std::{cmp::Ordering, ffi::OsStr}; +#[cfg(not(target_os = "wasi"))] use std::{ - cmp::Ordering, - ffi::OsStr, io::Read, iter, sync::mpsc::{Receiver, SyncSender, sync_channel}, @@ -21,11 +21,78 @@ use std::{ }; use uucore::error::UResult; +fn buffer_size(settings: &GlobalSettings) -> usize { + if settings.buffer_size < 100 * 1024 { + // when the buffer size is smaller than 100KiB we choose it instead of the default. + // this improves testability. + settings.buffer_size + } else { + 100 * 1024 + } +} + +/// Given the chunks of a file (in order), find the first pair of adjacent +/// lines that violates the requested ordering and report it as a +/// [`SortError::Disorder`]. +#[cfg(target_os = "wasi")] +fn check_chunks( + path: &OsStr, + settings: &GlobalSettings, + max_allowed_cmp: Ordering, + chunks: impl Iterator, +) -> UResult<()> { + let mut prev_chunk: Option = None; + let mut line_idx = 0; + for chunk in chunks { + line_idx += 1; + if let Some(prev_chunk) = &prev_chunk { + // Check if the first element of the new chunk is greater than the last + // element from the previous chunk + let prev_last = prev_chunk.lines().last().unwrap(); + let new_first = chunk.lines().first().unwrap(); + + if compare_by( + prev_last, + new_first, + settings, + prev_chunk.line_data(), + chunk.line_data(), + ) > max_allowed_cmp + { + return Err(SortError::Disorder { + file: path.to_owned(), + line_number: line_idx, + line: String::from_utf8_lossy(new_first.line).into_owned(), + silent: settings.check_silent, + } + .into()); + } + } + + for (a, b) in chunk.lines().iter().tuple_windows() { + line_idx += 1; + if compare_by(a, b, settings, chunk.line_data(), chunk.line_data()) > max_allowed_cmp { + return Err(SortError::Disorder { + file: path.to_owned(), + line_number: line_idx, + line: String::from_utf8_lossy(b.line).into_owned(), + silent: settings.check_silent, + } + .into()); + } + } + + prev_chunk = Some(chunk); + } + Ok(()) +} + /// Check if the file at `path` is ordered. /// /// # Returns /// /// The code we should exit with. +#[cfg(not(target_os = "wasi"))] pub fn check(path: &OsStr, settings: &GlobalSettings) -> UResult<()> { let max_allowed_cmp = if settings.unique { // If `unique` is enabled, the previous line must compare _less_ to the next one. @@ -42,13 +109,7 @@ pub fn check(path: &OsStr, settings: &GlobalSettings) -> UResult<()> { move || reader(file, &recycled_receiver, &loaded_sender, &settings) }); for _ in 0..2 { - let _ = recycled_sender.send(RecycledChunk::new(if settings.buffer_size < 100 * 1024 { - // when the buffer size is smaller than 100KiB we choose it instead of the default. - // this improves testability. - settings.buffer_size - } else { - 100 * 1024 - })); + let _ = recycled_sender.send(RecycledChunk::new(buffer_size(settings))); } let mut prev_chunk: Option = None; @@ -114,7 +175,27 @@ pub fn check(path: &OsStr, settings: &GlobalSettings) -> UResult<()> { result } +/// Check if the file at `path` is ordered. +/// +/// WASI has no thread support, so this reads every chunk up front on the +/// current thread instead of streaming them from a background reader thread. +/// +/// # Returns +/// +/// The code we should exit with. +#[cfg(target_os = "wasi")] +pub fn check(path: &OsStr, settings: &GlobalSettings) -> UResult<()> { + let max_allowed_cmp = if settings.unique { + Ordering::Less + } else { + Ordering::Equal + }; + let chunks = read_all_chunks(path, settings)?; + check_chunks(path, settings, max_allowed_cmp, chunks.into_iter()) +} + /// The function running on the reader thread. +#[cfg(not(target_os = "wasi"))] fn reader( mut file: Box, receiver: &Receiver, @@ -139,3 +220,33 @@ fn reader( } Ok(()) } + +/// Read every chunk of `path` up front, without any recycling or background +/// thread. Used on WASI, which has no thread support. +#[cfg(target_os = "wasi")] +fn read_all_chunks(path: &OsStr, settings: &GlobalSettings) -> UResult> { + let mut file = open(path)?; + let mut carry_over = vec![]; + let mut chunks = Vec::new(); + let (sender, receiver) = std::sync::mpsc::sync_channel(1); + loop { + let recycled = RecycledChunk::new(buffer_size(settings)); + let should_continue = chunks::read( + &sender, + recycled, + None, + &mut carry_over, + &mut file, + &mut std::iter::empty(), + settings.line_ending.into(), + settings, + )?; + while let Ok(chunk) = receiver.try_recv() { + chunks.push(chunk); + } + if !should_continue { + break; + } + } + Ok(chunks) +} diff --git a/src/uu/sort/src/ext_sort/wasi.rs b/src/uu/sort/src/ext_sort/wasi.rs index 50bd5f63033..d37c1f41cec 100644 --- a/src/uu/sort/src/ext_sort/wasi.rs +++ b/src/uu/sort/src/ext_sort/wasi.rs @@ -31,6 +31,13 @@ pub fn ext_sort( // moderately sized inputs; very large files may cause OOM. let mut input = Vec::new(); for file in files { + // Insert the separator between files whose preceding content doesn't + // already end with one; otherwise the last line of one file would + // merge with the first line of the next, e.g. "a\nb" + "b" -> + // "a\nbb" instead of "a\nb" + '\n' + "b". + if !input.is_empty() && input.last() != Some(&separator) { + input.push(separator); + } file?.read_to_end(&mut input)?; } if input.is_empty() { diff --git a/tests/by-util/test_sort.rs b/tests/by-util/test_sort.rs index 9dd0233b23c..a93397d8261 100644 --- a/tests/by-util/test_sort.rs +++ b/tests/by-util/test_sort.rs @@ -36,6 +36,7 @@ fn test_helper(file_name: &str, possible_args: &[&str]) { } #[test] +#[cfg_attr(wasi_runner, ignore)] fn test_buffer_sizes() { #[cfg(target_os = "linux")] let buffer_sizes = ["0", "50K", "50k", "1M", "100M", "0%", "10%"]; @@ -52,6 +53,8 @@ fn test_buffer_sizes() { .stdout_is_fixture("ext_sort.expected"); } + // The wasm guest is always a 32-bit target regardless of the host's + // pointer width, so these overflow there even when the host is 64-bit. #[cfg(not(target_pointer_width = "32"))] { let buffer_sizes = ["1000G", "10T"]; @@ -666,6 +669,7 @@ fn month_sort_input_expected(months: &[String]) -> (String, String) { #[test] #[cfg(unix)] +#[cfg_attr(wasi_runner, ignore = "WASI sandbox: locale database not visible")] fn test_month_sort_french_locale() { let locale = "fr_FR.UTF-8"; if !is_locale_available(locale) { @@ -697,6 +701,7 @@ fn test_month_sort_french_locale() { #[test] #[cfg(unix)] +#[cfg_attr(wasi_runner, ignore = "WASI sandbox: locale database not visible")] fn test_month_sort_hungarian_locale() { let locale = "hu_HU.UTF-8"; if !is_locale_available(locale) { @@ -728,6 +733,7 @@ fn test_month_sort_hungarian_locale() { /// E.g. "av ril" should NOT match "avril" — GNU treats it as unknown. #[test] #[cfg(unix)] +#[cfg_attr(wasi_runner, ignore = "WASI sandbox: locale database not visible")] fn test_month_sort_french_embedded_blanks() { let locale = "fr_FR.UTF-8"; if !is_locale_available(locale) { @@ -780,6 +786,7 @@ fn test_month_sort_french_embedded_blanks() { #[test] #[cfg(unix)] +#[cfg_attr(wasi_runner, ignore = "WASI sandbox: locale database not visible")] fn test_month_sort_japanese_locale() { let locale = "ja_JP.UTF-8"; if !is_locale_available(locale) { @@ -1099,6 +1106,10 @@ fn test_multiple_files() { } #[test] +#[cfg_attr( + wasi_runner, + ignore = "WASI: sort -m spawns real OS threads for multi-file merge, unsupported under wasmtime's default config" +)] fn test_merge_interleaved() { new_ucmd!() .arg("-m") @@ -1110,6 +1121,10 @@ fn test_merge_interleaved() { } #[test] +#[cfg_attr( + wasi_runner, + ignore = "WASI: sort -m spawns real OS threads for multi-file merge, unsupported under wasmtime's default config" +)] fn test_merge_preserves_long_lines() { use std::fmt::Write; @@ -1142,6 +1157,10 @@ fn test_merge_preserves_long_lines() { // receivers while the reader was still sending, and `chunks::read` unwraps that send. #[test] #[cfg(target_os = "linux")] +#[cfg_attr( + wasi_runner, + ignore = "WASI: sort -m spawns real OS threads for multi-file merge, unsupported under wasmtime's default config" +)] fn test_merge_write_error_does_not_panic() { use std::fs::File; @@ -1173,6 +1192,10 @@ fn test_merge_write_error_does_not_panic() { } #[test] +#[cfg_attr( + wasi_runner, + ignore = "WASI: sort -m spawns real OS threads for multi-file merge, unsupported under wasmtime's default config" +)] fn test_merge_unique() { new_ucmd!() .arg("-m") @@ -1188,6 +1211,10 @@ fn test_merge_unique() { } #[test] +#[cfg_attr( + wasi_runner, + ignore = "WASI: sort -m spawns real OS threads for multi-file merge, unsupported under wasmtime's default config" +)] fn test_merge_stable() { new_ucmd!() .arg("-m") @@ -1200,6 +1227,10 @@ fn test_merge_stable() { } #[test] +#[cfg_attr( + wasi_runner, + ignore = "WASI: sort -m spawns real OS threads for multi-file merge, unsupported under wasmtime's default config" +)] fn test_merge_reversed() { new_ucmd!() .arg("-m") @@ -1405,6 +1436,10 @@ fn test_compress() { #[test] #[cfg(any(target_os = "linux", target_os = "android"))] +#[cfg_attr( + wasi_runner, + ignore = "WASI: sort -m spawns real OS threads for multi-file merge, unsupported under wasmtime's default config" +)] fn test_compress_merge() { new_ucmd!() .args(&[ @@ -1428,6 +1463,10 @@ fn test_compress_merge() { #[test] #[cfg(not(target_os = "android"))] +#[cfg_attr( + wasi_runner, + ignore = "WASI: ext_sort bypasses --compress-program entirely (single-threaded in-memory path)" +)] fn test_compress_fail() { let result = new_ucmd!() .args(&[ @@ -1490,7 +1529,7 @@ fn test_batch_size_too_large() { "--batch-size argument '{large_batch_size}' too large" )); - #[cfg(target_os = "linux")] + #[cfg(all(target_os = "linux", not(wasi_runner)))] new_ucmd!() .arg(format!("--batch-size={large_batch_size}")) .fails_with_code(2) @@ -1498,6 +1537,10 @@ fn test_batch_size_too_large() { } #[test] +#[cfg_attr( + wasi_runner, + ignore = "WASI: sort -m spawns real OS threads for multi-file merge, unsupported under wasmtime's default config" +)] fn test_merge_batch_size() { new_ucmd!() .arg("--batch-size=2") @@ -1517,6 +1560,10 @@ fn test_merge_batch_size() { // TODO(#7542): Re-enable on Android once we figure out why setting limit is broken. // #[cfg(any(target_os = "linux", target_os = "android"))] #[cfg(target_os = "linux")] +#[cfg_attr( + wasi_runner, + ignore = "WASI: sort -m spawns real OS threads for multi-file merge, unsupported under wasmtime's default config" +)] fn test_merge_batch_size_with_limit() { use rlimit::Resource; // Currently need... @@ -1612,6 +1659,10 @@ fn test_verifies_files_after_keys() { #[test] #[cfg(unix)] +#[cfg_attr( + wasi_runner, + ignore = "WASI sandbox: host paths (/dev/random) not visible" +)] fn test_verifies_input_files() { new_ucmd!() .args(&["/dev/random", "nonexistent_file"]) @@ -1629,6 +1680,10 @@ fn test_separator_null() { } #[test] +#[cfg_attr( + wasi_runner, + ignore = "WASI: sort -m spawns real OS threads, unsupported under wasmtime's default config" +)] fn test_output_is_input() { let input = "a\nb\nc\n"; let (at, mut ucmd) = at_and_ucmd!(); @@ -1643,6 +1698,10 @@ fn test_output_is_input() { #[test] #[cfg(unix)] +#[cfg_attr( + wasi_runner, + ignore = "WASI sandbox: host paths (/dev/null) not visible" +)] fn test_output_device() { new_ucmd!() .args(&["-o", "/dev/null"]) @@ -1651,6 +1710,10 @@ fn test_output_device() { } #[test] +#[cfg_attr( + wasi_runner, + ignore = "WASI: sort -m spawns real OS threads for multi-file merge, unsupported under wasmtime's default config" +)] fn test_merge_empty_input() { new_ucmd!() .args(&["-m", "empty.txt"]) @@ -1676,6 +1739,7 @@ fn test_wrong_args_exit_code() { #[test] #[cfg(unix)] +#[cfg_attr(wasi_runner, ignore = "WASI: no signal support (SIGINT)")] fn test_tmp_files_deleted_on_sigint() { use rand::{RngExt as _, SeedableRng, rngs::SmallRng}; use rustix::process::{Pid, Signal, kill_process}; @@ -1761,6 +1825,7 @@ fn test_args_check_conflict() { #[cfg(target_os = "linux")] #[test] +#[cfg_attr(wasi_runner, ignore = "WASI P2: /dev/full filesystem not available")] fn test_failed_write_is_reported() { new_ucmd!() .pipe_in("hello") @@ -1863,6 +1928,10 @@ fn test_files0_from_empty() { #[test] #[cfg(unix)] +#[cfg_attr( + wasi_runner, + ignore = "WASI preview2: error message text for this OS error differs from native Unix" +)] fn test_files0_from_non_utf8_name() { new_ucmd!() .args(&["--files0-from", "-"]) @@ -1873,6 +1942,10 @@ fn test_files0_from_non_utf8_name() { #[test] #[cfg(unix)] +#[cfg_attr( + wasi_runner, + ignore = "WASI: error message text for this OS error differs from native Unix" +)] fn test_files0_read_error() { new_ucmd!() .args(&["--files0-from", "."]) @@ -1883,6 +1956,7 @@ fn test_files0_read_error() { #[cfg(unix)] #[test] // Test for GNU tests/sort/sort-files0-from.pl "empty-non-regular" +#[cfg_attr(wasi_runner, ignore = "WASI sandbox: host paths not visible")] fn test_files0_from_empty_non_regular() { new_ucmd!() .args(&["--files0-from", "/dev/null"]) @@ -1969,6 +2043,10 @@ fn test_files0_from_2a() { #[test] // Test for GNU tests/sort/sort-files0-from.pl "non-utf8" #[cfg(all(unix, not(target_os = "macos")))] +#[cfg_attr( + wasi_runner, + ignore = "WASI: preopened directories reject non-UTF-8 filenames" +)] fn test_files0_from_non_utf8() { use std::os::unix::ffi::OsStringExt; let (at, mut ucmd) = at_and_ucmd!(); @@ -2957,6 +3035,7 @@ fn test_locale_collation_utf8() { } #[test] +#[cfg_attr(wasi_runner, ignore = "WASI sandbox: locale database not visible")] fn test_locale_interleaved_en_us_utf8() { // Test case for issue: locale-based collation support // In en_US.UTF-8, lowercase and uppercase letters should interleave @@ -3029,6 +3108,7 @@ fn test_locale_with_ignore_case_flag() { } #[test] +#[cfg_attr(wasi_runner, ignore = "WASI sandbox: locale database not visible")] fn test_locale_complex_utf8_sorting() { // More complex test with mixed case and special characters // In en_US.UTF-8, should respect locale collation rules @@ -3053,6 +3133,7 @@ fn test_locale_posix_sort_debug_message() { } #[test] +#[cfg_attr(wasi_runner, ignore = "WASI sandbox: locale database not visible")] fn test_locale_utf8_sort_debug_message() { new_ucmd!() .env("LC_ALL", "en_US.UTF-8") @@ -3073,7 +3154,7 @@ fn test_failed_to_set_locale_debug_message() { result.stderr_contains("text ordering performed using simple byte comparison"); - #[cfg(all(target_os = "linux", target_env = "gnu"))] + #[cfg(all(target_os = "linux", target_env = "gnu", not(wasi_runner)))] result.stderr_contains("failed to set locale"); } @@ -3095,6 +3176,7 @@ e f 5436 down data path1 path2 path3 path4 path5\n"; } #[test] +#[cfg_attr(wasi_runner, ignore = "WASI sandbox: locale database not visible")] fn test_consistent_sorting_with_i18n_collate() { // Regression test for issue #11980 // Lexicographic fallback sorting for equal sorting keys for 01 and 0_1