From d3468ac2ad76068714e7856120884c481ade12fb Mon Sep 17 00:00:00 2001
From: AkiPoh <59505647+AkiPoh@users.noreply.github.com>
Date: Sun, 1 Nov 2026 06:12:41 +0200
Subject: [PATCH 1/2] Fix \w not matching UTF-8 continuation bytes
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

\w previously only included the leading bytes of multibyte sequences
(0xC2-0xF4), so a match ended in the middle of a non-ASCII character.
Because \w+ is a single non-backtracking charset instruction, rules
like java's /(\w+)\s*\(/ (for method definition highlighting) then
failed outright, leaving non-ASCII identifiers unhighlighted.

Hence this commit adds the continuation bytes (0x80-0xBF) to the \w
matching in regex.rs, to fix this issue.

See example that previously went unhighlighted in Java:

```java
// "merkkienMäärä" was previously unhighlighted due to containing
// non-ASCII characters. After this commit this too is highlighted.
public static int merkkienMäärä (String merkkijono) {
}

// In comparison "numberOfCharacters" got previously highlighted
// because it only included solely ASCII-characters.
public static int numberOfCharacters (String text) {
}
```
---
 crates/lsh/src/compiler/regex.rs | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/crates/lsh/src/compiler/regex.rs b/crates/lsh/src/compiler/regex.rs
index 4b9126bc931..60e21a91e1d 100644
--- a/crates/lsh/src/compiler/regex.rs
+++ b/crates/lsh/src/compiler/regex.rs
@@ -37,8 +37,9 @@
 //!
 //! # Gotchas
 //!
-//! - `\w` includes bytes 0xC2-0xF4 (UTF-8 leading bytes) so that it can consume multibyte characters.
-//!   This isn't Unicode-correct but works for identifiers in most programming languages.
+//! - `\w` includes bytes 0xC2-0xF4 (UTF-8 leading bytes) and 0x80-0xBF (UTF-8
+//!   continuation bytes), so that it can consume multibyte characters. This
+//!   isn't Unicode-correct but works for identifiers in most programming languages.
 //! - We don't create loops to keep the IR generation and optimization simple.
 //!   This means that e.g. (a|b)+ is not supported. For now that's fine.
 //! - The `parse()` function wires its generated IR into the provided destination nodes.
@@ -50,9 +51,11 @@ use super::charset::Charset;
 use super::ir::*;
 use crate::runtime::Register;
 
-// 0xC2-0xF4 are UTF-8 leading bytes for multibyte sequences. Including them lets
-// `\w+` consume entire multibyte characters, which is important for identifiers
-// containing non-ASCII letters (e.g., `naïve`, `café`).
+// 0xC2-0xF4 are UTF-8 leading bytes and 0x80-0xBF are UTF-8 continuation bytes.
+// Including both lets `\w+` consume entire multibyte characters, which is
+// important for identifiers containing non-ASCII letters (e.g., `naïve`,
+// `café`). This isn't Unicode-correct but works for identifiers in most
+// programming languages.
 const ASCII_WORD_CHARSET: Charset = {
     let mut charset = Charset::no();
     charset.set_range(b'0'..=b'9', true);
@@ -60,6 +63,7 @@ const ASCII_WORD_CHARSET: Charset = {
     charset.set_range(b'_'..=b'_', true);
     charset.set_range(b'a'..=b'z', true);
     charset.set_range(0xC2..=0xF4, true);
+    charset.set_range(0x80..=0xBF, true);
     charset
 };
 

From 627e191c5711d26e9bf00be5e3e899bf76593d1e Mon Sep 17 00:00:00 2001
From: Leonard Hecker <leonard@hecker.io>
Date: Mon, 2 Nov 2026 20:59:26 +0100
Subject: [PATCH 2/2] Improve comments

---
 crates/lsh/src/compiler/regex.rs | 15 +++++----------
 1 file changed, 5 insertions(+), 10 deletions(-)

diff --git a/crates/lsh/src/compiler/regex.rs b/crates/lsh/src/compiler/regex.rs
index 60e21a91e1d..418c6f66d0e 100644
--- a/crates/lsh/src/compiler/regex.rs
+++ b/crates/lsh/src/compiler/regex.rs
@@ -37,9 +37,9 @@
 //!
 //! # Gotchas
 //!
-//! - `\w` includes bytes 0xC2-0xF4 (UTF-8 leading bytes) and 0x80-0xBF (UTF-8
-//!   continuation bytes), so that it can consume multibyte characters. This
-//!   isn't Unicode-correct but works for identifiers in most programming languages.
+//! - `\w` includes bytes 0xC2-0xF4 (UTF-8 leading bytes) and 0x80-0xBF
+//!   (UTF-8 continuation bytes), so that it can consume multibyte characters.
+//!   This isn't Unicode-correct but works for identifiers in most programming languages.
 //! - We don't create loops to keep the IR generation and optimization simple.
 //!   This means that e.g. (a|b)+ is not supported. For now that's fine.
 //! - The `parse()` function wires its generated IR into the provided destination nodes.
@@ -51,19 +51,14 @@ use super::charset::Charset;
 use super::ir::*;
 use crate::runtime::Register;
 
-// 0xC2-0xF4 are UTF-8 leading bytes and 0x80-0xBF are UTF-8 continuation bytes.
-// Including both lets `\w+` consume entire multibyte characters, which is
-// important for identifiers containing non-ASCII letters (e.g., `naïve`,
-// `café`). This isn't Unicode-correct but works for identifiers in most
-// programming languages.
 const ASCII_WORD_CHARSET: Charset = {
     let mut charset = Charset::no();
     charset.set_range(b'0'..=b'9', true);
     charset.set_range(b'A'..=b'Z', true);
     charset.set_range(b'_'..=b'_', true);
     charset.set_range(b'a'..=b'z', true);
-    charset.set_range(0xC2..=0xF4, true);
-    charset.set_range(0x80..=0xBF, true);
+    charset.set_range(0xC2..=0xF4, true); // UTF8 leading bytes
+    charset.set_range(0x80..=0xBF, true); // UTF8 continuation bytes
     charset
 };