From ce57a49ffbb49d60f4357dc29fec8c1a10068d32 Mon Sep 17 00:00:00 2001 From: wtcpython <1762993226@qq.com> Date: Sun, 13 Sep 2026 16:04:49 +0800 Subject: [PATCH 1/2] uucore, expr: centralize regex in uucore --- Cargo.lock | 2 +- Cargo.toml | 6 +- fuzz/Cargo.lock | 2 +- src/uu/expr/Cargo.toml | 3 +- src/uu/expr/locales/en-US.ftl | 7 - src/uu/expr/locales/fr-FR.ftl | 7 - src/uu/expr/src/diagnostics.rs | 8 +- src/uu/expr/src/expr.rs | 16 +- src/uu/expr/src/syntax_tree.rs | 371 ++--------------- src/uucore/Cargo.toml | 6 + src/uucore/locales/errors/en-US.ftl | 10 + src/uucore/locales/errors/fr-FR.ftl | 10 + src/uucore/src/lib/features.rs | 2 + src/uucore/src/lib/features/regex.rs | 587 +++++++++++++++++++++++++++ src/uucore/src/lib/lib.rs | 2 + tests/by-util/test_expr.rs | 41 ++ 16 files changed, 697 insertions(+), 383 deletions(-) create mode 100644 src/uucore/src/lib/features/regex.rs diff --git a/Cargo.lock b/Cargo.lock index 72b667c7574..fe7e846ce3e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3535,7 +3535,6 @@ version = "0.13.0" dependencies = [ "clap", "codspeed-divan-compat", - "fancy-regex", "fluent", "num-bigint", "num-traits", @@ -4499,6 +4498,7 @@ dependencies = [ "digest 0.11.3", "dns-lookup", "dunce", + "fancy-regex", "fluent", "fluent-syntax", "glob", diff --git a/Cargo.toml b/Cargo.toml index 7ce0a2c10be..b85816acf0e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -374,11 +374,7 @@ num-bigint = "0.4.4" num-integer = "0.1.47" num-prime = "0.5.0" num-traits = "0.2.19" -fancy-regex = { version = "0.19.2", default-features = false, features = [ - "std", - "unicode", - "leftmost_longest", -] } +fancy-regex = { version = "0.19.2", default-features = false } os_display = "0.1.3" parse_datetime = "0.16.0" phf = "0.14.0" diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index 2d6d4a29634..7c3f5394fe7 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -1870,7 +1870,6 @@ name = "uu_expr" version = "0.13.0" dependencies = [ "clap", - "fancy-regex", "fluent", "num-bigint", "num-traits", @@ -1985,6 +1984,7 @@ dependencies = [ "data-encoding-macro", "digest 0.11.3", "dunce", + "fancy-regex", "fluent", "fluent-syntax", "glob", diff --git a/src/uu/expr/Cargo.toml b/src/uu/expr/Cargo.toml index 83fed4854a3..dfbdb73d6d4 100644 --- a/src/uu/expr/Cargo.toml +++ b/src/uu/expr/Cargo.toml @@ -19,8 +19,7 @@ doctest = false clap = { workspace = true } num-bigint = { workspace = true } num-traits = { workspace = true } -fancy-regex = { workspace = true, default-features = false } -uucore = { workspace = true, features = ["i18n-collator"] } +uucore = { workspace = true, features = ["i18n-collator", "regex"] } thiserror = { workspace = true } fluent = { workspace = true } diff --git a/src/uu/expr/locales/en-US.ftl b/src/uu/expr/locales/en-US.ftl index 6c600aee0ba..b281e0b26e4 100644 --- a/src/uu/expr/locales/en-US.ftl +++ b/src/uu/expr/locales/en-US.ftl @@ -95,15 +95,8 @@ expr-error-missing-argument = syntax error: missing argument after { $arg } expr-error-non-integer-argument = non-integer argument expr-error-missing-operand = missing operand expr-error-division-by-zero = division by zero -expr-error-invalid-regex-expression = Invalid regex expression expr-error-expected-closing-brace-after = syntax error: expecting ')' after { $arg } expr-error-expected-closing-brace-instead-of = syntax error: expecting ')' instead of { $arg } -expr-error-unmatched-opening-parenthesis = Unmatched ( or \( -expr-error-unmatched-closing-parenthesis = Unmatched ) or \) -expr-error-unmatched-opening-brace = Unmatched {"\\{"} -expr-error-invalid-bracket-content = Invalid content of {"\\{\\}"} -expr-error-trailing-backslash = Trailing backslash -expr-error-too-big-range-quantifier-index = Regular expression too big expr-error-match-utf8 = match does not support invalid UTF-8 encoding in { $arg } # Diagnostic labels, used when errors are rendered with a source snippet diff --git a/src/uu/expr/locales/fr-FR.ftl b/src/uu/expr/locales/fr-FR.ftl index 8a376fa0186..793a2e20e27 100644 --- a/src/uu/expr/locales/fr-FR.ftl +++ b/src/uu/expr/locales/fr-FR.ftl @@ -95,15 +95,8 @@ expr-error-missing-argument = erreur de syntaxe : argument manquant après { $ar expr-error-non-integer-argument = argument non entier expr-error-missing-operand = opérande manquant expr-error-division-by-zero = division par zéro -expr-error-invalid-regex-expression = Expression regex invalide expr-error-expected-closing-brace-after = erreur de syntaxe : ')' attendu après { $arg } expr-error-expected-closing-brace-instead-of = erreur de syntaxe : ')' attendu au lieu de { $arg } -expr-error-unmatched-opening-parenthesis = Parenthèse ouvrante ( ou \( non appariée -expr-error-unmatched-closing-parenthesis = Parenthèse fermante ) ou \) non appariée -expr-error-unmatched-opening-brace = Accolade ouvrante {"\\{"} non appariée -expr-error-invalid-bracket-content = Contenu invalide de {"\\{\\}"} -expr-error-trailing-backslash = Barre oblique inverse en fin -expr-error-too-big-range-quantifier-index = Expression régulière trop grande expr-error-match-utf8 = match ne supporte pas l'encodage UTF-8 invalide dans { $arg } # Étiquettes de diagnostic, utilisées quand les erreurs sont rendues avec un extrait diff --git a/src/uu/expr/src/diagnostics.rs b/src/uu/expr/src/diagnostics.rs index 3a2893c9a8f..4284269920d 100644 --- a/src/uu/expr/src/diagnostics.rs +++ b/src/uu/expr/src/diagnostics.rs @@ -81,13 +81,7 @@ fn locate(snapshot: &Snapshot, err: &ExprError, at: &FailurePoint) -> Option return None, }; diff --git a/src/uu/expr/src/expr.rs b/src/uu/expr/src/expr.rs index d84eab9a525..4144d0fe8a3 100644 --- a/src/uu/expr/src/expr.rs +++ b/src/uu/expr/src/expr.rs @@ -42,26 +42,14 @@ pub enum ExprError { MissingOperand, #[error("{}", translate!("expr-error-division-by-zero"))] DivisionByZero, - #[error("{}", translate!("expr-error-invalid-regex-expression"))] - InvalidRegexExpression, #[error("{}", translate!("expr-error-expected-closing-brace-after", "arg" => _0.quote()))] ExpectedClosingBraceAfter(String), #[error("{}", translate!("expr-error-expected-closing-brace-instead-of", "arg" => _0.quote()))] ExpectedClosingBraceInsteadOf(String), - #[error("{}", translate!("expr-error-unmatched-opening-parenthesis"))] - UnmatchedOpeningParenthesis, - #[error("{}", translate!("expr-error-unmatched-closing-parenthesis"))] - UnmatchedClosingParenthesis, - #[error("{}", translate!("expr-error-unmatched-opening-brace"))] - UnmatchedOpeningBrace, - #[error("{}", translate!("expr-error-invalid-bracket-content"))] - InvalidBracketContent, - #[error("{}", translate!("expr-error-trailing-backslash"))] - TrailingBackslash, - #[error("{}", translate!("expr-error-too-big-range-quantifier-index"))] - TooBigRangeQuantifierIndex, #[error("{}", translate!("expr-error-match-utf8", "arg" => _0.quote()))] UnsupportedNonUtf8Match(String), + #[error(transparent)] + Regex(#[from] uucore::regex::RegexError), } impl UError for ExprError { diff --git a/src/uu/expr/src/syntax_tree.rs b/src/uu/expr/src/syntax_tree.rs index d3b42b4a3a8..c32d74b0eb3 100644 --- a/src/uu/expr/src/syntax_tree.rs +++ b/src/uu/expr/src/syntax_tree.rs @@ -7,9 +7,9 @@ use std::{cell::Cell, collections::BTreeMap}; -use fancy_regex::{Regex, RegexBuilder}; use num_bigint::BigInt; use num_traits::ToPrimitive; +use uucore::regex::{Regex, RegexBuilder, bre_to_ere}; use crate::{ ExprError, ExprResult, @@ -155,260 +155,6 @@ impl StringOp { } } -/// Check if regex pattern character iterator is at the end of a regex expression or subexpression -fn is_end_of_expression(pattern_chars: &I) -> bool -where - I: Iterator + Clone, -{ - let mut pattern_chars_clone = pattern_chars.clone(); - match pattern_chars_clone.next() { - Some('\\') => matches!(pattern_chars_clone.next(), Some(')' | '|')), - None => true, // No characters left - _ => false, - } -} - -/// Check if regex pattern character iterator is at the start of a valid range quantifier. -/// The iterator's start position is expected to be after the opening brace. -/// Range quantifier ends to closing brace. -/// -/// # Examples of valid range quantifiers -/// -/// - `r"\{3\}"` -/// - `r"\{3,\}"` -/// - `r"\{,6\}"` -/// - `r"\{3,6\}"` -/// - `r"\{,\}"` -fn verify_range_quantifier(pattern_chars: &I) -> Result<(), ExprError> -where - I: Iterator + Clone, -{ - let mut pattern_chars_clone = pattern_chars.clone().peekable(); - if pattern_chars_clone.peek().is_none() { - return Err(ExprError::UnmatchedOpeningBrace); - } - - // Parse the string between braces - let mut quantifier = String::new(); - let mut prev = '\0'; - let mut curr_is_escaped = false; - while let Some(curr) = pattern_chars_clone.next() { - curr_is_escaped = prev == '\\' && !curr_is_escaped; - if curr_is_escaped && curr == '}' { - break; - } - if pattern_chars_clone.peek().is_none() { - return Err(ExprError::UnmatchedOpeningBrace); - } - if prev != '\0' { - quantifier.push(prev); - } - prev = curr; - } - - // Check if parsed quantifier is valid - let re = Regex::new(r"^([0-9]*,[0-9]*|[0-9]+)$").expect("valid regular expression"); - if let Ok(Some(captures)) = re.captures(&quantifier) { - let matched = captures.get(0).map_or("", |m| m.as_str()); - match matched.split_once(',') { - Some(("", "")) => Ok(()), - Some((x, "") | ("", x)) if x.parse::().is_ok() => Ok(()), - Some((_, "") | ("", _)) => Err(ExprError::TooBigRangeQuantifierIndex), - Some((f, l)) => match (f.parse::(), l.parse::()) { - (Ok(f), Ok(l)) if f > l => Err(ExprError::InvalidBracketContent), - (Ok(_), Ok(_)) => Ok(()), - _ => Err(ExprError::TooBigRangeQuantifierIndex), - }, - None if matched.parse::().is_ok() => Ok(()), - None => Err(ExprError::TooBigRangeQuantifierIndex), - } - } else { - Err(ExprError::InvalidBracketContent) - } -} - -/// Check for errors in a supplied regular expression -/// -/// GNU coreutils shows messages for invalid regular expressions -/// differently from standard regex engines. -/// This method attempts to do these checks manually in one pass -/// through the regular expression. -/// -/// This method is not comprehensively checking all cases in which -/// a regular expression could be invalid; any cases not caught will -/// result in a [`ExprError::InvalidRegexExpression`]. This method is -/// intended to just identify a few situations for which GNU coreutils -/// has specific error messages. -fn check_posix_regex_errors(pattern: &str) -> ExprResult<()> { - let mut escaped_parens: u64 = 0; - let mut prev = '\0'; - let mut curr_is_escaped = false; - - for curr in pattern.chars() { - curr_is_escaped = prev == '\\' && !curr_is_escaped; - match (curr_is_escaped, curr) { - (true, '(') => escaped_parens += 1, - (true, ')') => { - escaped_parens = escaped_parens - .checked_sub(1) - .ok_or(ExprError::UnmatchedClosingParenthesis)?; - } - _ => {} - } - prev = curr; - } - - match escaped_parens { - 0 => Ok(()), - _ => Err(ExprError::UnmatchedOpeningParenthesis), - } -} - -/// Transpile the input pattern from BRE syntax to ERE for fancy-regex -fn transpile_bre_to_ere(pattern_str: &str) -> ExprResult { - let mut re_string = String::with_capacity(pattern_str.len() + 8); - let mut pattern_chars = pattern_str.chars().peekable(); - let mut prev = '\0'; - let mut prev_is_escaped = false; - let mut in_bracket = false; - let mut bracket_start_idx = 0; - let mut is_start_of_expression = true; - let mut after_anchor_caret = false; - - // All patterns are anchored so they begin with a caret (^) - if pattern_chars.peek() != Some(&'^') { - re_string.push('^'); - } - - while let Some(curr) = pattern_chars.next() { - let curr_is_escaped = prev == '\\' && !prev_is_escaped; - let mut next_is_start = false; - let mut next_after_anchor = false; - - if in_bracket { - let is_first = re_string.len() == bracket_start_idx + 1 - || (re_string.ends_with("[^") && re_string.len() == bracket_start_idx + 2); - if curr == ']' && !curr_is_escaped && !is_first { - in_bracket = false; - } - re_string.push(curr); - } else { - match curr { - '[' if !curr_is_escaped => { - in_bracket = true; - bracket_start_idx = re_string.len(); - re_string.push('['); - } - // In BRE, '(', ')', '|', '+', '?', '{', '}' are literal by default, - // and become operators only when escaped. ERE has the exact opposite convention. - '(' | ')' | '|' | '+' | '?' | '{' | '}' => { - if curr_is_escaped { - if re_string.ends_with('\\') { - re_string.pop(); - } - match curr { - '(' | '|' => { - re_string.push(curr); - next_is_start = true; - } - '+' | '?' => { - if is_start_of_expression || after_anchor_caret { - re_string.push('\\'); - } - re_string.push(curr); - } - '{' => { - // Handle '{' literally at the start of an expression - if is_start_of_expression || after_anchor_caret { - re_string.push_str(r"\{"); - } else { - // Check if the following section is a valid range quantifier - verify_range_quantifier(&pattern_chars)?; - re_string.push('{'); - // Set the lower bound of range quantifier to 0 if it is missing - if pattern_chars.peek() == Some(&',') { - re_string.push('0'); - } - } - } - _ => re_string.push(curr), // ')' and '}' - } - } else { - // Unescaped metacharacter in BRE -> literal in ERE - re_string.push('\\'); - re_string.push(curr); - } - } - '*' => { - if curr_is_escaped { - re_string.push('*'); - } else if is_start_of_expression || after_anchor_caret { - re_string.push_str(r"\*"); - } else { - re_string.push('*'); - } - } - // Character class negation "[^a]" - // Explicitly escaped caret "\^" - '^' => { - if curr_is_escaped { - re_string.push('^'); - } else if is_start_of_expression { - re_string.push('^'); - next_after_anchor = true; - } else if prev == '[' && !prev_is_escaped { - re_string.push('^'); - } else { - re_string.push_str(r"\^"); - } - } - '$' if !curr_is_escaped && !is_end_of_expression(&pattern_chars) => { - re_string.push_str(r"\$"); - } - '<' if curr_is_escaped => { - if re_string.ends_with('\\') { - re_string.pop(); - } - re_string.push_str(r"\b(?=\w)"); - } - '>' if curr_is_escaped => { - if re_string.ends_with('\\') { - re_string.pop(); - } - re_string.push_str(r"\b(?<=\w)"); - } - '\\' if !curr_is_escaped => { - if pattern_chars.peek().is_none() { - return Err(ExprError::TrailingBackslash); - } - // Carry the expression-start / after-anchor state over the - // backslash so the escaped character is still treated as the - // first token of a (sub)expression. - next_is_start = is_start_of_expression; - next_after_anchor = after_anchor_caret; - re_string.push('\\'); - } - _ => { - if curr_is_escaped - && !"123456789.*^$[]\\wWsSbB".contains(curr) - && re_string.ends_with('\\') - { - re_string.pop(); - } - re_string.push(curr); - } - } - } - - is_start_of_expression = next_is_start; - after_anchor_caret = next_after_anchor; - prev_is_escaped = curr_is_escaped; - prev = curr; - } - - Ok(re_string) -} - /// Build a regex from a pattern string with locale-aware encoding fn build_regex(pattern_bytes: Vec) -> ExprResult { use uucore::i18n::UEncoding; @@ -421,15 +167,17 @@ fn build_regex(pattern_bytes: Vec) -> ExprResult { .unwrap_or_else(|_| String::from_utf8_lossy(&pattern_bytes).into()), UEncoding::Ascii => pattern_bytes.iter().map(|&b| b as char).collect(), }; - check_posix_regex_errors(&pattern_str)?; - let re_string = transpile_bre_to_ere(&pattern_str)?; + let re_string = bre_to_ere(&pattern_str, true)?; - RegexBuilder::new(&format!("(?s){re_string}")) + let regex = RegexBuilder::new(&format!("(?s){re_string}")) .oniguruma_mode(true) .leftmost_longest(true) + .seek(true) .build() - .map_err(|_| ExprError::InvalidRegexExpression) + .map_err(uucore::regex::RegexError::from)?; + + Ok(regex) } /// Find matches in the input using the compiled regex @@ -1115,12 +863,11 @@ pub fn is_truthy(s: &NumOrStr) -> bool { #[cfg(test)] mod test { - use crate::syntax_tree::verify_range_quantifier; use crate::{ExprError, ExprResult}; + use uucore::regex::RegexError; use super::{ - AstNode, AstNodeInner, BinOp, MaybeNonUtf8Str, NumericOp, RelationOp, StringOp, - check_posix_regex_errors, get_next_id, + AstNode, AstNodeInner, BinOp, MaybeNonUtf8Str, NumericOp, RelationOp, StringOp, get_next_id, }; /// Parse an expression, discarding how far the parser got. @@ -1337,78 +1084,6 @@ mod test { assert_eq!(result.eval_as_string(), b"0"); } - #[test] - fn check_regex_valid() { - assert!(check_posix_regex_errors(r"(a+b) \(a* b\)").is_ok()); - } - - #[test] - fn check_regex_simple_repeating_pattern() { - assert!(check_posix_regex_errors(r"\(a+b\)\{4\}").is_ok()); - } - - #[test] - fn check_regex_missing_closing() { - assert_eq!( - check_posix_regex_errors(r"\(abc"), - Err(ExprError::UnmatchedOpeningParenthesis) - ); - } - - #[test] - fn check_regex_missing_opening() { - assert_eq!( - check_posix_regex_errors(r"abc\)"), - Err(ExprError::UnmatchedClosingParenthesis) - ); - } - - #[test] - fn test_is_valid_range_quantifier() { - assert!(verify_range_quantifier(&"3\\}".chars()).is_ok()); - assert!(verify_range_quantifier(&"3,\\}".chars()).is_ok()); - assert!(verify_range_quantifier(&",6\\}".chars()).is_ok()); - assert!(verify_range_quantifier(&"3,6\\}".chars()).is_ok()); - assert!(verify_range_quantifier(&",\\}".chars()).is_ok()); - assert!(verify_range_quantifier(&"32767\\}anything".chars()).is_ok()); - assert_eq!( - verify_range_quantifier(&"\\{3,6\\}".chars()), - Err(ExprError::InvalidBracketContent) - ); - assert_eq!( - verify_range_quantifier(&"\\}".chars()), - Err(ExprError::InvalidBracketContent) - ); - assert_eq!( - verify_range_quantifier(&"".chars()), - Err(ExprError::UnmatchedOpeningBrace) - ); - assert_eq!( - verify_range_quantifier(&"3".chars()), - Err(ExprError::UnmatchedOpeningBrace) - ); - assert_eq!( - verify_range_quantifier(&"3,".chars()), - Err(ExprError::UnmatchedOpeningBrace) - ); - assert_eq!( - verify_range_quantifier(&",6".chars()), - Err(ExprError::UnmatchedOpeningBrace) - ); - assert_eq!( - verify_range_quantifier(&"3,6".chars()), - Err(ExprError::UnmatchedOpeningBrace) - ); - assert_eq!( - verify_range_quantifier(&",".chars()), - Err(ExprError::UnmatchedOpeningBrace) - ); - assert_eq!( - verify_range_quantifier(&"32768\\}".chars()), - Err(ExprError::TooBigRangeQuantifierIndex) - ); - } - #[test] fn test_evaluate_match_expression_basic() { use super::evaluate_match_expression; @@ -1573,23 +1248,29 @@ mod test { let result = evaluate_match_expression(b"hello".to_vec(), b"\\(hello".to_vec()); assert!(matches!( result, - Err(ExprError::UnmatchedOpeningParenthesis) + Err(ExprError::Regex(RegexError::UnmatchedOpeningParenthesis)) )); // Unmatched closing parenthesis let result = evaluate_match_expression(b"hello".to_vec(), b"hello\\)".to_vec()); assert!(matches!( result, - Err(ExprError::UnmatchedClosingParenthesis) + Err(ExprError::Regex(RegexError::UnmatchedClosingParenthesis)) )); // Trailing backslash let result = evaluate_match_expression(b"hello".to_vec(), b"hello\\".to_vec()); - assert!(matches!(result, Err(ExprError::TrailingBackslash))); + assert!(matches!( + result, + Err(ExprError::Regex(RegexError::TrailingBackslash)) + )); // Invalid bracket content let result = evaluate_match_expression(b"hello".to_vec(), b"a\\{invalid\\}".to_vec()); - assert!(matches!(result, Err(ExprError::InvalidBracketContent))); + assert!(matches!( + result, + Err(ExprError::Regex(RegexError::InvalidBracketContent)) + )); } #[test] @@ -1631,10 +1312,14 @@ mod test { use super::evaluate_match_expression; // This test verifies leftmost-longest (POSIX) match semantics. - // Pattern `(a|ab)` against `ab` should capture the longest alternative (`ab`) + // Pattern `\(a\|ab\)` against `ab` should capture the longest alternative (`ab`), // not the first (`a`). let result = evaluate_match_expression(b"ab".to_vec(), br"\(a\|ab\)".to_vec()).unwrap(); assert_eq!(result.eval_as_string(), b"ab"); + + // `aaaaa\|a*` against `aaaaaa` should match all 6 characters + let result = evaluate_match_expression(b"aaaaaa".to_vec(), br"aaaaa\|a*".to_vec()).unwrap(); + assert_eq!(result.eval_as_string(), b"6"); } #[test] @@ -1657,6 +1342,14 @@ mod test { let result = evaluate_match_expression(b"b".to_vec(), br"b\>".to_vec()).unwrap(); assert_eq!(result.eval_as_string(), b"1"); + // Beginning of buffer \` + let result = evaluate_match_expression(b"start".to_vec(), br"\`start".to_vec()).unwrap(); + assert_eq!(result.eval_as_string(), b"5"); + + // End of buffer \' + let result = evaluate_match_expression(b"end".to_vec(), br"end\'".to_vec()).unwrap(); + assert_eq!(result.eval_as_string(), b"3"); + // Escaped caret with quantifier \^* let result = evaluate_match_expression(b"^".to_vec(), br"\^*".to_vec()).unwrap(); assert_eq!(result.eval_as_string(), b"1"); diff --git a/src/uucore/Cargo.toml b/src/uucore/Cargo.toml index 3f8841b6555..0f11a16ef90 100644 --- a/src/uucore/Cargo.toml +++ b/src/uucore/Cargo.toml @@ -25,6 +25,11 @@ clap = { workspace = true } data-encoding = { workspace = true, optional = true } data-encoding-macro = { workspace = true, optional = true } dunce = { workspace = true, optional = true } +fancy-regex = { workspace = true, optional = true, features = [ + "std", + "unicode", + "leftmost_longest", +] } glob = { workspace = true, optional = true } itertools = { workspace = true, optional = true } jiff = { workspace = true, optional = true, features = [ @@ -193,6 +198,7 @@ process = ["libc", "nix/user", "rustix/process", "windows-sys"] proc-info = ["tty", "walkdir"] quoting-style = ["i18n-common"] ranges = [] +regex = ["dep:fancy-regex"] ringbuffer = [] safe-copy = ["rustix/fs"] safe-traversal = ["libc", "nix/fs", "nix/dir", "nix/user"] diff --git a/src/uucore/locales/errors/en-US.ftl b/src/uucore/locales/errors/en-US.ftl index 76d73436aeb..f5b6c3636ad 100644 --- a/src/uucore/locales/errors/en-US.ftl +++ b/src/uucore/locales/errors/en-US.ftl @@ -54,3 +54,13 @@ size-diag-help-syntax = a size is a number and an optional unit: K, M, G and so # in its own words. range-diag-label-too-large = this number is too large range-diag-label-inverted = this range ends before it starts + +# Regex parsing messages (expr, grep, sed, ...) +regex-error-unmatched-opening-parenthesis = Unmatched ( or \( +regex-error-unmatched-closing-parenthesis = Unmatched ) or \) +regex-error-trailing-backslash = Trailing backslash +regex-error-unmatched-opening-brace = Unmatched {"\\{"} +regex-error-invalid-bracket-content = Invalid content of {"\\{\\}"} +regex-error-too-big-range-quantifier-index = Regular expression too big +regex-error-invalid-character-class-name = Invalid character class name +regex-error-compilation-failed = Invalid regular expression: { $error } diff --git a/src/uucore/locales/errors/fr-FR.ftl b/src/uucore/locales/errors/fr-FR.ftl index 03b9eff60fe..8b3193e25b8 100644 --- a/src/uucore/locales/errors/fr-FR.ftl +++ b/src/uucore/locales/errors/fr-FR.ftl @@ -54,3 +54,13 @@ size-diag-help-syntax = une taille est un nombre suivi d'une unité facultative # compte, donc chaque utilitaire le dit avec ses propres mots. range-diag-label-too-large = ce nombre est trop grand range-diag-label-inverted = cet intervalle se termine avant de commencer + +# Messages d'analyse regex (expr, grep, sed, ...) +regex-error-unmatched-opening-parenthesis = Parenthèse ouvrante ( ou \( non appariée +regex-error-unmatched-closing-parenthesis = Parenthèse fermante ) ou \) non appariée +regex-error-trailing-backslash = Barre oblique inverse en fin +regex-error-unmatched-opening-brace = Accolade ouvrante {"\\{"} non appariée +regex-error-invalid-bracket-content = Contenu invalide de {"\\{\\}"} +regex-error-too-big-range-quantifier-index = Expression régulière trop grande +regex-error-invalid-character-class-name = Nom de classe de caractères invalide +regex-error-compilation-failed = Expression régulière invalide : { $error } diff --git a/src/uucore/src/lib/features.rs b/src/uucore/src/lib/features.rs index e8b82567ad5..c2b342e2605 100644 --- a/src/uucore/src/lib/features.rs +++ b/src/uucore/src/lib/features.rs @@ -55,6 +55,8 @@ pub mod parser; pub mod quoting_style; #[cfg(feature = "ranges")] pub mod ranges; +#[cfg(feature = "regex")] +pub mod regex; #[cfg(feature = "ringbuffer")] pub mod ringbuffer; #[cfg(feature = "sum")] diff --git a/src/uucore/src/lib/features/regex.rs b/src/uucore/src/lib/features/regex.rs new file mode 100644 index 00000000000..28b405df85b --- /dev/null +++ b/src/uucore/src/lib/features/regex.rs @@ -0,0 +1,587 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +//! Unified regular expression infrastructure for uutils. +//! +//! Provides a clean abstraction layer around [`fancy_regex`] with support for +//! POSIX Basic Regular Expressions (BRE) transpilation, GNU extensions, +//! and POSIX leftmost-longest matching semantics. + +use crate::translate; +pub use fancy_regex::{Regex, RegexBuilder}; + +/// Errors encountered when compiling or transpiling regular expressions. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum RegexError { + #[error("{}", translate!("regex-error-unmatched-opening-parenthesis"))] + UnmatchedOpeningParenthesis, + + #[error("{}", translate!("regex-error-unmatched-closing-parenthesis"))] + UnmatchedClosingParenthesis, + + #[error("{}", translate!("regex-error-trailing-backslash"))] + TrailingBackslash, + + #[error("{}", translate!("regex-error-unmatched-opening-brace"))] + UnmatchedOpeningBrace, + + #[error("{}", translate!("regex-error-invalid-bracket-content"))] + InvalidBracketContent, + + #[error("{}", translate!("regex-error-too-big-range-quantifier-index"))] + TooBigRangeQuantifierIndex, + + #[error("{}", translate!("regex-error-invalid-character-class-name"))] + InvalidCharacterClassName, + + #[error("{}", translate!("regex-error-compilation-failed", "error" => _0))] + CompilationFailed(String), +} + +impl crate::error::UError for RegexError { + fn code(&self) -> i32 { + 2 + } +} + +impl From for RegexError { + fn from(err: fancy_regex::Error) -> Self { + Self::CompilationFailed(err.to_string()) + } +} + +/// Map POSIX character class name (e.g. `"alpha"`, `"digit"`) to its Unicode property equivalent. +pub fn map_posix_class(name: &str) -> Option<&'static str> { + match name { + "alpha" => Some(r"\p{Alphabetic}"), + "lower" => Some(r"\p{Lowercase}"), + "upper" => Some(r"\p{Uppercase}"), + "alnum" => Some(r"\p{Alphabetic}0-9"), + "space" => Some(r"\p{White_Space}"), + "blank" => Some(r"\t\p{Zs}"), + "cntrl" => Some(r"\p{Control}"), + "digit" => Some("0-9"), + "xdigit" => Some("0-9A-Fa-f"), + "punct" => Some(r"\p{Punctuation}"), + "graph" => Some(r"\P{C}&&\P{Z}"), + "print" => Some(r"\P{C}"), + _ => None, + } +} + +/// Check if a regex character iterator is at the start of a valid range quantifier (`\{m,n\}`). +/// +/// The iterator's start position is expected to be immediately after the opening brace. +fn verify_range_quantifier(pattern_chars: &I) -> Result<(), RegexError> +where + I: Iterator + Clone, +{ + let mut pattern_chars_clone = pattern_chars.clone().peekable(); + if pattern_chars_clone.peek().is_none() { + return Err(RegexError::UnmatchedOpeningBrace); + } + + // Parse the string between braces + let mut quantifier = String::new(); + let mut prev = '\0'; + let mut curr_is_escaped = false; + while let Some(curr) = pattern_chars_clone.next() { + curr_is_escaped = prev == '\\' && !curr_is_escaped; + if curr_is_escaped && curr == '}' { + break; + } + if pattern_chars_clone.peek().is_none() { + return Err(RegexError::UnmatchedOpeningBrace); + } + if prev != '\0' { + quantifier.push(prev); + } + prev = curr; + } + + // Check if parsed quantifier is valid + let re = Regex::new(r"^([0-9]*,[0-9]*|[0-9]+)$").expect("valid regular expression"); + if let Ok(Some(captures)) = re.captures(&quantifier) { + let matched = captures.get(0).map_or("", |m| m.as_str()); + match matched.split_once(',') { + Some(("", "")) => Ok(()), + Some((x, "") | ("", x)) if x.parse::().is_ok() => Ok(()), + Some((_, "") | ("", _)) => Err(RegexError::TooBigRangeQuantifierIndex), + Some((f, l)) => match (f.parse::(), l.parse::()) { + (Ok(f), Ok(l)) if f > l => Err(RegexError::InvalidBracketContent), + (Ok(_), Ok(_)) => Ok(()), + _ => Err(RegexError::TooBigRangeQuantifierIndex), + }, + None if matched.parse::().is_ok() => Ok(()), + None => Err(RegexError::TooBigRangeQuantifierIndex), + } + } else { + Err(RegexError::InvalidBracketContent) + } +} + +/// Check for errors in a supplied regular expression +/// +/// GNU coreutils shows messages for invalid regular expressions +/// differently from standard regex engines. +/// This method attempts to do these checks manually in one pass +/// through the regular expression. +/// +/// This method is not comprehensively checking all cases in which +/// a regular expression could be invalid; any cases not caught will +/// fall through to `fancy-regex` compilation. This method is intended to +/// just identify a few situations for which GNU coreutils has specific +/// error messages. +fn check_posix_regex_errors(pattern: &str) -> Result<(), RegexError> { + let mut escaped_parens: u64 = 0; + let mut prev = '\0'; + let mut curr_is_escaped = false; + + for curr in pattern.chars() { + curr_is_escaped = prev == '\\' && !curr_is_escaped; + match (curr_is_escaped, curr) { + (true, '(') => escaped_parens += 1, + (true, ')') => { + escaped_parens = escaped_parens + .checked_sub(1) + .ok_or(RegexError::UnmatchedClosingParenthesis)?; + } + _ => {} + } + prev = curr; + } + + match escaped_parens { + 0 => Ok(()), + _ => Err(RegexError::UnmatchedOpeningParenthesis), + } +} + +/// Check if regex pattern character iterator is at the end of a regex expression or subexpression +fn is_end_of_expression(pattern_chars: &I) -> bool +where + I: Iterator + Clone, +{ + let mut pattern_chars_clone = pattern_chars.clone(); + match pattern_chars_clone.next() { + Some('\\') => matches!(pattern_chars_clone.next(), Some(')' | '|')), + None => true, // No characters left + _ => false, + } +} + +/// Transpile a POSIX Basic Regular Expression (BRE) into Extended Regular Expression (ERE) +/// compatible with `fancy-regex`. +/// +/// If `anchored` is `true`, the pattern is implicitly anchored at the start (`^`), as required by POSIX `expr`. +/// Otherwise, standard unanchored BRE transpilation is performed (e.g. for `grep` and `sed`). +pub fn bre_to_ere(pattern_str: &str, anchored: bool) -> Result { + check_posix_regex_errors(pattern_str)?; + + let mut re_string = String::with_capacity(pattern_str.len() + 8); + let mut pattern_chars = pattern_str.chars().peekable(); + let mut prev = '\0'; + let mut prev_is_escaped = false; + let mut in_bracket = false; + let mut is_start_of_expression = true; + let mut after_anchor_caret = false; + + if anchored && pattern_chars.peek() != Some(&'^') { + re_string.push('^'); + } + + while let Some(curr) = pattern_chars.next() { + let curr_is_escaped = prev == '\\' && !prev_is_escaped; + let mut next_is_start = false; + let mut next_after_anchor = false; + + if in_bracket { + if curr == '[' && pattern_chars.peek() == Some(&':') { + pattern_chars.next(); + let mut name = String::new(); + let mut closed = false; + while let Some(c) = pattern_chars.next() { + if c == ':' && pattern_chars.peek() == Some(&']') { + pattern_chars.next(); + closed = true; + break; + } + name.push(c); + } + if closed { + if let Some(unicode_class) = map_posix_class(&name) { + re_string.push_str(unicode_class); + prev = ']'; + prev_is_escaped = false; + continue; + } + return Err(RegexError::InvalidCharacterClassName); + } + re_string.push_str("[:"); + re_string.push_str(&name); + prev = name.chars().last().unwrap_or(':'); + prev_is_escaped = false; + continue; + } + if curr == ']' && re_string.ends_with(|c| c != '\\' && c != '[' && c != '^') { + in_bracket = false; + } + re_string.push(curr); + prev = curr; + prev_is_escaped = false; + continue; + } + + match curr { + '[' if !curr_is_escaped => { + in_bracket = true; + re_string.push('['); + } + // In BRE, '(', ')', '|', '+', '?', '{', '}' are literal by default, + // and become operators only when escaped. ERE has the exact opposite convention. + '(' | ')' | '|' | '+' | '?' | '{' | '}' => { + if curr_is_escaped { + if re_string.ends_with('\\') { + re_string.pop(); + } + match curr { + '(' | '|' => { + re_string.push(curr); + next_is_start = true; + } + '+' | '?' => { + if is_start_of_expression || after_anchor_caret { + re_string.push('\\'); + } + re_string.push(curr); + } + '{' => { + // Handle '{' literally at the start of an expression + if is_start_of_expression || after_anchor_caret { + re_string.push_str(r"\{"); + } else { + // Check if the following section is a valid range quantifier + verify_range_quantifier(&pattern_chars)?; + re_string.push('{'); + // Set the lower bound of range quantifier to 0 if it is missing + if pattern_chars.peek() == Some(&',') { + re_string.push('0'); + } + } + } + _ => re_string.push(curr), // ')' and '}' + } + } else { + // Unescaped metacharacter in BRE -> literal in ERE + re_string.push('\\'); + re_string.push(curr); + } + } + '*' => { + if curr_is_escaped { + re_string.push('*'); + } else if is_start_of_expression || after_anchor_caret { + re_string.push_str(r"\*"); + } else { + re_string.push('*'); + } + } + // Character class negation "[^a]" + // Explicitly escaped caret "\^" + '^' => { + if curr_is_escaped { + re_string.push('^'); + } else if is_start_of_expression { + re_string.push('^'); + next_after_anchor = true; + } else if prev == '[' && !prev_is_escaped { + re_string.push('^'); + } else { + re_string.push_str(r"\^"); + } + } + '$' if !curr_is_escaped && !is_end_of_expression(&pattern_chars) => { + re_string.push_str(r"\$"); + } + '`' if curr_is_escaped => { + if re_string.ends_with('\\') { + re_string.pop(); + } + re_string.push_str(r"\A"); + } + '\'' if curr_is_escaped => { + if re_string.ends_with('\\') { + re_string.pop(); + } + re_string.push_str(r"\z"); + } + '<' if curr_is_escaped => { + if re_string.ends_with('\\') { + re_string.pop(); + } + re_string.push_str(r"\b(?=\w)"); + } + '>' if curr_is_escaped => { + if re_string.ends_with('\\') { + re_string.pop(); + } + re_string.push_str(r"\b(?<=\w)"); + } + '\\' if !curr_is_escaped => { + if pattern_chars.peek().is_none() { + return Err(RegexError::TrailingBackslash); + } + // Carry the expression-start / after-anchor state over the + // backslash so the escaped character is still treated as the + // first token of a (sub)expression. + next_is_start = is_start_of_expression; + next_after_anchor = after_anchor_caret; + re_string.push('\\'); + } + _ => { + if curr_is_escaped + && !"123456789.*^$[]\\wWsSbB".contains(curr) + && re_string.ends_with('\\') + { + re_string.pop(); + } + re_string.push(curr); + } + } + + is_start_of_expression = next_is_start; + after_anchor_caret = next_after_anchor; + prev_is_escaped = curr_is_escaped; + prev = curr; + } + Ok(re_string) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_bre_groups() { + assert_eq!(bre_to_ere(r"foo\(bar\)baz", false).unwrap(), "foo(bar)baz"); + } + + #[test] + fn test_bre_intervals() { + assert_eq!(bre_to_ere(r"ab\{1,3\}c", false).unwrap(), "ab{1,3}c"); + assert_eq!(bre_to_ere(r"ab\{,3\}c", false).unwrap(), "ab{0,3}c"); + } + + #[test] + fn test_bre_alternation() { + assert_eq!(bre_to_ere(r"foo\|bar", false).unwrap(), "foo|bar"); + } + + #[test] + fn test_bre_literal_specials() { + assert_eq!( + bre_to_ere("a+b?c|d(e)f{g}", false).unwrap(), + r"a\+b\?c\|d\(e\)f\{g\}" + ); + } + + #[test] + fn test_bre_anchors() { + assert_eq!(bre_to_ere("^foo$", false).unwrap(), "^foo$"); + assert_eq!(bre_to_ere("a^b$c", false).unwrap(), r"a\^b\$c"); + } + + #[test] + fn test_bre_word_boundaries() { + assert_eq!( + bre_to_ere(r"\", false).unwrap(), + r"\b(?=\w)word\b(?<=\w)" + ); + } + + #[test] + fn test_bre_buffer_anchors() { + assert_eq!(bre_to_ere(r"\`start", false).unwrap(), r"\Astart"); + assert_eq!(bre_to_ere(r"end\'", false).unwrap(), r"end\z"); + } + + #[test] + fn test_bre_posix_classes() { + assert_eq!( + bre_to_ere(r"[[:alpha:]]", false).unwrap(), + r"[\p{Alphabetic}]" + ); + assert_eq!(bre_to_ere(r"[[:digit:]]", false).unwrap(), r"[0-9]"); + assert_eq!( + bre_to_ere(r"[^[:lower:]]", false).unwrap(), + r"[^\p{Lowercase}]" + ); + assert_eq!( + bre_to_ere(r"[[:alpha:][:digit:]]", false).unwrap(), + r"[\p{Alphabetic}0-9]" + ); + assert_eq!(bre_to_ere(r"[]a]", false).unwrap(), "[]a]"); + assert_eq!(bre_to_ere(r"[^]a]", false).unwrap(), "[^]a]"); + + // Unicode match test: [[:alpha:]] matches 'é' + let transpiled = bre_to_ere(r"[[:alpha:]]", true).unwrap(); + let re = Regex::new(&format!("(?s){transpiled}")).unwrap(); + assert!(re.is_match("é").unwrap()); + } + + #[test] + fn test_bre_invalid_posix_class() { + assert_eq!( + bre_to_ere(r"[[:bogus:]]", false).unwrap_err(), + RegexError::InvalidCharacterClassName + ); + assert_eq!( + bre_to_ere(r"[[:123:]]", false).unwrap_err(), + RegexError::InvalidCharacterClassName + ); + } + + #[test] + fn test_bre_backrefs() { + assert_eq!(bre_to_ere(r"\(foo\)\1", false).unwrap(), r"(foo)\1"); + } + + #[test] + fn test_bre_leading_quantifier() { + assert_eq!(bre_to_ere("*foo", false).unwrap(), r"\*foo"); + assert_eq!(bre_to_ere(r"\(*foo\)", false).unwrap(), r"(\*foo)"); + assert_eq!(bre_to_ere(r"a\|*b", false).unwrap(), r"a|\*b"); + } + + #[test] + fn test_bre_anchored() { + assert_eq!(bre_to_ere("abc", true).unwrap(), "^abc"); + assert_eq!(bre_to_ere("^abc", true).unwrap(), "^abc"); + assert_eq!(bre_to_ere(r"\(foo\)", true).unwrap(), "^(foo)"); + } + + #[test] + fn test_posix_errors() { + assert_eq!( + bre_to_ere(r"\(foo", false).unwrap_err(), + RegexError::UnmatchedOpeningParenthesis + ); + assert_eq!( + bre_to_ere(r"foo\)", false).unwrap_err(), + RegexError::UnmatchedClosingParenthesis + ); + assert_eq!( + bre_to_ere(r"foo\", false).unwrap_err(), + RegexError::TrailingBackslash + ); + assert_eq!( + bre_to_ere(r"foo\{1", false).unwrap_err(), + RegexError::UnmatchedOpeningBrace + ); + assert_eq!( + bre_to_ere(r"foo\{5,2\}", false).unwrap_err(), + RegexError::InvalidBracketContent + ); + } + + #[test] + fn test_leftmost_longest_semantics() { + // POSIX leftmost-longest requires (a|ab) against "ab" to match "ab" (longest), not "a". + let transpiled = bre_to_ere(r"\(a\|ab\)", false).unwrap(); + let re = RegexBuilder::new(&transpiled) + .oniguruma_mode(true) + .leftmost_longest(true) + .seek(true) + .build() + .unwrap(); + + let caps = re.captures("ab").unwrap().expect("should match"); + assert_eq!(caps.get(1).unwrap().as_str(), "ab"); + + // "aaaaa|a*" against "aaaaaa" should match all 6 "a"s + let transpiled2 = bre_to_ere(r"aaaaa\|a*", false).unwrap(); + let re2 = RegexBuilder::new(&transpiled2) + .oniguruma_mode(true) + .leftmost_longest(true) + .seek(true) + .build() + .unwrap(); + + let m = re2.find("aaaaaa").unwrap().expect("should match"); + assert_eq!(m.as_str(), "aaaaaa"); + } + + #[test] + fn check_regex_valid() { + assert!(check_posix_regex_errors(r"(a+b) \(a* b\)").is_ok()); + } + + #[test] + fn check_regex_simple_repeating_pattern() { + assert!(check_posix_regex_errors(r"\(a+b\)\{4\}").is_ok()); + } + + #[test] + fn check_regex_missing_closing() { + assert_eq!( + check_posix_regex_errors(r"\(abc"), + Err(RegexError::UnmatchedOpeningParenthesis) + ); + } + + #[test] + fn check_regex_missing_opening() { + assert_eq!( + check_posix_regex_errors(r"abc\)"), + Err(RegexError::UnmatchedClosingParenthesis) + ); + } + + #[test] + fn test_is_valid_range_quantifier() { + assert!(verify_range_quantifier(&"3\\}".chars()).is_ok()); + assert!(verify_range_quantifier(&"3,\\}".chars()).is_ok()); + assert!(verify_range_quantifier(&",6\\}".chars()).is_ok()); + assert!(verify_range_quantifier(&"3,6\\}".chars()).is_ok()); + assert!(verify_range_quantifier(&",\\}".chars()).is_ok()); + assert!(verify_range_quantifier(&"32767\\}anything".chars()).is_ok()); + assert_eq!( + verify_range_quantifier(&"\\{3,6\\}".chars()), + Err(RegexError::InvalidBracketContent) + ); + assert_eq!( + verify_range_quantifier(&"\\}".chars()), + Err(RegexError::InvalidBracketContent) + ); + assert_eq!( + verify_range_quantifier(&"".chars()), + Err(RegexError::UnmatchedOpeningBrace) + ); + assert_eq!( + verify_range_quantifier(&"3".chars()), + Err(RegexError::UnmatchedOpeningBrace) + ); + assert_eq!( + verify_range_quantifier(&"3,".chars()), + Err(RegexError::UnmatchedOpeningBrace) + ); + assert_eq!( + verify_range_quantifier(&",6".chars()), + Err(RegexError::UnmatchedOpeningBrace) + ); + assert_eq!( + verify_range_quantifier(&"3,6".chars()), + Err(RegexError::UnmatchedOpeningBrace) + ); + assert_eq!( + verify_range_quantifier(&",".chars()), + Err(RegexError::UnmatchedOpeningBrace) + ); + assert_eq!( + verify_range_quantifier(&"32768\\}".chars()), + Err(RegexError::TooBigRangeQuantifierIndex) + ); + } +} diff --git a/src/uucore/src/lib/lib.rs b/src/uucore/src/lib/lib.rs index 53781b7f92a..3014b69a993 100644 --- a/src/uucore/src/lib/lib.rs +++ b/src/uucore/src/lib/lib.rs @@ -74,6 +74,8 @@ pub use crate::features::parser; pub use crate::features::quoting_style; #[cfg(feature = "ranges")] pub use crate::features::ranges; +#[cfg(feature = "regex")] +pub use crate::features::regex; #[cfg(feature = "ringbuffer")] pub use crate::features::ringbuffer; #[cfg(feature = "sum")] diff --git a/tests/by-util/test_expr.rs b/tests/by-util/test_expr.rs index fed1729f755..242ba873adf 100644 --- a/tests/by-util/test_expr.rs +++ b/tests/by-util/test_expr.rs @@ -505,6 +505,47 @@ fn test_regex_leftmost_longest_match_semantics() { .args(&["ab", ":", "a\\|ab"]) .succeeds() .stdout_only("2\n"); + + new_ucmd!() + .args(&["aaaaaa", ":", "aaaaa\\|a*"]) + .succeeds() + .stdout_only("6\n"); + + new_ucmd!() + .args(&["ab", ":", "\\(a\\|ab\\)"]) + .succeeds() + .stdout_only("ab\n"); + + new_ucmd!() + .args(&["aaaaaa", ":", "\\(aaaaa\\|a*\\)"]) + .succeeds() + .stdout_only("aaaaaa\n"); +} + +#[test] +#[cfg_attr(wasi_runner, ignore = "WASI: no locale data, every locale is C")] +fn test_regex_posix_character_classes() { + new_ucmd!() + .args(&["é", ":", "[[:alpha:]]"]) + .succeeds() + .stdout_only("1\n"); + + new_ucmd!() + .env("LC_ALL", "en_US.UTF-8") + .args(&["é", ":", "[[:lower:]]"]) + .succeeds() + .stdout_only("1\n"); + + new_ucmd!() + .env("LC_ALL", "en_US.UTF-8") + .args(&["É", ":", "[[:upper:]]"]) + .succeeds() + .stdout_only("1\n"); + + new_ucmd!() + .args(&["a", ":", "[[:bogus:]]"]) + .fails_with_code(2) + .stderr_contains("Invalid character class name"); } #[test] From 73b9fbc3cd9f5a9e170bb7ea300870ca59f45749 Mon Sep 17 00:00:00 2001 From: CodSpeed Bot Date: Wed, 16 Sep 2026 14:23:06 +0000 Subject: [PATCH 2/2] uucore: reach the error-only strings only for the ids they define The error-only resource is parsed on the first lookup that misses every ordinary bundle. That is not only an error path: a binary that cannot resolve its own strings misses on every id it asks for, and a bench binary calling `uumain` directly is exactly that. Every benchmark in the tree therefore parsed the whole resource -- 106k of the 319k instructions hostname_basic measures, and 16 allocations it never frees. Growing the resource consequently charged every benchmark, whether or not the utility can reach the new strings. The eight regex messages added here moved hostname_basic from 318,823 to 335,034 instructions (+5.1%) and every utility's peak memory by 4.2 KB, in utilities that never compile a regex. Generate the ids the resource defines at build time and check that before building the bundle, so a lookup that cannot be in there costs a match instead of a parse: hostname_basic instructions main (94811cc) 318823 this branch 335034 this branch, this commit 212632 Measured with `codspeed run --mode simulation -- cargo codspeed run -p uu_hostname`. The expr benchmarks are unchanged (1,448,685 against 1,448,761 on main), and expr still reports its regex diagnostics, in English and in French. --- src/uucore/build.rs | 85 +++++++++++++++++++++++++++++++ src/uucore/src/lib/mods/locale.rs | 9 ++++ 2 files changed, 94 insertions(+) diff --git a/src/uucore/build.rs b/src/uucore/build.rs index a9a7d09e6b6..3d28feb472f 100644 --- a/src/uucore/build.rs +++ b/src/uucore/build.rs @@ -54,10 +54,76 @@ pub fn main() -> Result<(), Box> { writeln!(embedded_file, " }}")?; writeln!(embedded_file, "}}")?; + generate_error_locale_ids(&mut embedded_file)?; + embedded_file.flush()?; Ok(()) } +/// Generate the lookup telling whether an id belongs to the error-only strings. +/// +/// Reaching those strings costs a parse of the whole resource, so it is only +/// worth doing for an id the resource actually defines. Which ids those are is +/// known at compile time, so the check itself costs nothing. +/// +/// # Errors +/// +/// Returns an error if `CARGO_MANIFEST_DIR` is not set, if the resource exists +/// but cannot be read, or if writing to `embedded_file` fails. +fn generate_error_locale_ids(embedded_file: &mut File) -> Result<(), Box> { + use std::fs; + + let path = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?).join("locales/errors/en-US.ftl"); + let content = if path.is_file() { + println!("cargo:rerun-if-changed={}", path.display()); + fs::read_to_string(&path)? + } else { + String::new() + }; + let ids = message_ids(&content); + + writeln!(embedded_file)?; + writeln!( + embedded_file, + "/// Whether the error-only strings define `id`." + )?; + writeln!(embedded_file, "fn is_error_locale_id(id: &str) -> bool {{")?; + if ids.is_empty() { + writeln!(embedded_file, " let _ = id;")?; + writeln!(embedded_file, " false")?; + } else { + writeln!(embedded_file, " matches!(")?; + writeln!(embedded_file, " id,")?; + for (i, id) in ids.iter().enumerate() { + let separator = if i == 0 { ' ' } else { '|' }; + writeln!(embedded_file, " {separator} \"{id}\"")?; + } + writeln!(embedded_file, " )")?; + } + writeln!(embedded_file, "}}")?; + + Ok(()) +} + +/// The ids of the messages `content` defines, in the order they appear. +fn message_ids(content: &str) -> Vec<&str> { + content + .lines() + .filter_map(|line| { + let id = line.split_once('=')?.0.trim_end(); + // A message starts in the first column: an indented line continues + // the message before it, and a leading '-' makes it a term, which + // no lookup ever asks for by name. + id.starts_with(|c: char| c.is_ascii_alphanumeric()) + .then_some(id) + .filter(|id| { + id.chars() + .all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_') + }) + }) + .collect() +} + /// Get the project root directory /// /// # Errors @@ -460,6 +526,25 @@ where mod tests { use super::*; + #[test] + fn message_ids_are_the_ones_starting_a_line() { + let content = concat!( + "# a comment = not a message\n", + "\n", + "first-id = a value\n", + "second-id = a value with an = in it\n", + " a continuation = still the second value\n", + "-a-term = not asked for by name\n", + "third_id = another value\n", + ); + + assert_eq!( + message_ids(content), + vec!["first-id", "second-id", "third_id"] + ); + assert!(message_ids("").is_empty()); + } + #[test] fn consumer_lock_file_found_above_target_dir() { let tmp = std::env::temp_dir().join(format!("uucore-build-{}", std::process::id())); diff --git a/src/uucore/src/lib/mods/locale.rs b/src/uucore/src/lib/mods/locale.rs index 0b49087bc8e..3e384477eb3 100644 --- a/src/uucore/src/lib/mods/locale.rs +++ b/src/uucore/src/lib/mods/locale.rs @@ -125,6 +125,15 @@ fn errors_message( id: &str, args: Option<&FluentArgs>, ) -> Option { + // Only an id this resource defines is worth that parse. Lookups reach here + // whenever they miss every ordinary bundle, which is not only error paths: + // a binary that cannot find its own strings misses on all of them, and a + // bench binary calling `uumain` directly is exactly that. The ids come + // from the English resource, the one every locale is a translation of. + if !is_error_locale_id(id) { + return None; + } + ERRORS_BUNDLE.with(|cell| { let bundle = cell.get_or_init(|| build_errors_bundle(locales)).as_ref()?; let message = bundle.get_message(id)?.value()?;