From 2f2ee39fa7187e60e669e8fb7e868dc4c80275fc Mon Sep 17 00:00:00 2001 From: tirdesh Date: Mon, 2 Nov 2026 21:18:03 -0500 Subject: [PATCH] expr: match valid characters before an invalid UTF-8 byte In a UTF-8 locale, find_match ran the regex on a Latin-1 mapping of the bytes whenever the left operand was not valid UTF-8. With no capture group that branch returned "0" whether or not the regex had matched, the same value as the no-match path, so it could never report a match. With a group the mapping let . match the invalid byte itself, so captures ran past it. GNU matches nothing against the invalid sequence but matches the valid characters in front of it as usual. Run the regex on the longest valid prefix instead, which gives that behaviour and lets both cases share the existing UTF-8 path. test_invalid_utf8_with_high_bytes_before_capture pinned the old mapping: it expected ".\(abc\)" to capture "abc" out of "\xFFabc", which needs . to match the invalid byte. The match is anchored, so GNU reports no match there and exits 1, the same way it does for the equivalent shape already covered by test_m6. The test now expects that in a UTF-8 locale and keeps the octet behaviour in a single byte one. --- src/uu/expr/src/syntax_tree.rs | 61 +++++++++++++++------------------- tests/by-util/test_expr.rs | 26 +++++++++++++++ 2 files changed, 53 insertions(+), 34 deletions(-) diff --git a/src/uu/expr/src/syntax_tree.rs b/src/uu/expr/src/syntax_tree.rs index 3bbfc54f6da..9d0d0cf873d 100644 --- a/src/uu/expr/src/syntax_tree.rs +++ b/src/uu/expr/src/syntax_tree.rs @@ -439,36 +439,23 @@ fn find_match(regex: Regex, left_bytes: Vec) -> String { // Match against the input using the appropriate encoding match encoding { UEncoding::Utf8 => { - // In UTF-8 locale, check if input is valid UTF-8 - if let Ok(left_str) = std::str::from_utf8(&left_bytes) { - // Valid UTF-8, match as UTF-8 - if let Ok(Some(caps)) = regex.captures(left_str) { - return if has_captures { - // Get first capture group - caps.get(1) - .map_or(String::new(), |m| m.as_str().to_string()) - } else { - // Count characters in the match - caps.get(0).unwrap().as_str().chars().count().to_string() - }; - } - } else { - // Invalid UTF-8 in UTF-8 locale: match on Latin-1 byte mapping - let left_str: String = left_bytes.iter().map(|&b| b as char).collect(); - if let Ok(Some(caps)) = regex.captures(&left_str) { - if has_captures { - if let Some(m) = caps.get(1) { - let bytes: Vec = m.as_str().chars().map(|c| c as u8).collect(); - // Return empty string for invalid UTF-8 capture in UTF-8 locale - if let Ok(s) = String::from_utf8(bytes) { - return s; - } - } - return String::new(); - } - // No capture groups - return 0 for invalid UTF-8 in UTF-8 locale - return "0".to_string(); - } + // Nothing matches an invalid byte sequence, but the valid characters + // in front of it still match as usual, so run the regex on the + // longest valid prefix. The slice up to `valid_up_to` is valid UTF-8 + // by construction. + let left_str = match std::str::from_utf8(&left_bytes) { + Ok(left_str) => left_str, + Err(e) => std::str::from_utf8(&left_bytes[..e.valid_up_to()]).unwrap_or_default(), + }; + if let Ok(Some(caps)) = regex.captures(left_str) { + return if has_captures { + // Get first capture group + caps.get(1) + .map_or(String::new(), |m| m.as_str().to_string()) + } else { + // Count characters in the match + caps.get(0).unwrap().as_str().chars().count().to_string() + }; } } UEncoding::Ascii => { @@ -1685,13 +1672,19 @@ mod test { use super::evaluate_match_expression; use uucore::i18n::{UEncoding, get_locale_encoding}; - // Input contains non-UTF-8 byte >= 0x80 (0xFF) before a valid ASCII capture + // A high byte (0xFF) in front of a valid ASCII capture. The match is anchored, + // so in a UTF-8 locale the leading `.` has to match the invalid byte, nothing + // does, and the match fails; in a single byte locale it is an ordinary octet. let result = evaluate_match_expression(vec![0xff, b'a', b'b', b'c'], br".\(abc\)".to_vec()).unwrap(); - assert_eq!(result.eval_as_string(), b"abc"); + match get_locale_encoding() { + UEncoding::Utf8 => assert_eq!(result.eval_as_string(), b""), + UEncoding::Ascii => assert_eq!(result.eval_as_string(), b"abc"), + } - // In UTF-8 locale, capture group containing invalid UTF-8 returns empty string; - // in ASCII/C locale (e.g. WASI default), raw bytes are captured. + // The same input with a greedy group: the valid prefix in front of the invalid + // byte is empty, so the UTF-8 locale captures nothing, while the single byte + // locale (e.g. WASI default) takes every octet. let result = evaluate_match_expression(vec![0xff, b'a', b'b', b'c'], br"\(.*\)".to_vec()).unwrap(); match get_locale_encoding() { diff --git a/tests/by-util/test_expr.rs b/tests/by-util/test_expr.rs index 7c1dca7310b..30cabf76664 100644 --- a/tests/by-util/test_expr.rs +++ b/tests/by-util/test_expr.rs @@ -2072,6 +2072,32 @@ mod expr_multibyte_arithmetic { check_test_case(args, tc); } } + + // An invalid sequence is matched by nothing, but the valid characters in + // front of it still match as usual. + #[test] + fn test_m8() { + let args: &[&[u8]] = &[b"match", b"/etc/x\xFF", b"/"]; + + let cases = &[TestCase::FR.out("1"), TestCase::C.out("1")]; + + for tc in cases { + check_test_case(args, tc); + } + } + + // The same with a group: in the multibyte locale the capture stops at the + // invalid byte, while the single byte locale takes it as an octet. + #[test] + fn test_m9() { + let args: &[&[u8]] = &[b"match", b"a/../\xFF", b"\\(.*\\)"]; + + let cases = &[TestCase::FR.out("a/../"), TestCase::C.out(b"a/../\xFF")]; + + for tc in cases { + check_test_case(args, tc); + } + } } #[test]