Improve support for emojis+cursor w/grapheme clustering

.github/workflows/test.yaml

@@ -32,7 +32,9 @@ jobs:
           uvx typos .
       - name: Unit test
         run: |
-          uvx --with . --with pytest coverage run -m pytest tests/
+          uvx --with . --with pytest coverage[toml] run -m pytest tests/
+          uvx coverage[toml] combine
+          uvx coverage[toml] report
       - name: Type Checking
         if: ${{ matrix.python-version != '3.8' }}
         run: |

pyproject.toml

@@ -22,7 +22,7 @@ classifiers = [
 ]
 requires-python = ">=3.8"
 dependencies = [
-    "wcwidth>=0.1.4",
+    "wcwidth>=0.5.0",
 ]
 
 [project.urls]
@@ -86,6 +86,9 @@ extend-ignore-re = [
     # Lorem ipsum.
     "Nam",
     "varius",
+    # Partial words in grapheme clustering tests (niño, café).
+    "nin",
+    "caf",
 ]
 
 locale = 'en-us'  # US English.
@@ -118,6 +121,10 @@ warn_return_any = true
 warn_unused_configs = true
 warn_unused_ignores = true
 
+[tool.coverage.run]
+source = ["src/prompt_toolkit"]
+parallel = true
+
 [build-system]
 requires = ["setuptools>=68"]
 build-backend = "setuptools.build_meta"

src/prompt_toolkit/buffer.py

@@ -18,6 +18,8 @@
 from functools import wraps
 from typing import Any, Callable, Coroutine, Iterable, TypeVar, cast
 
+import wcwidth
+
 from .application.current import get_app
 from .application.run_in_terminal import run_in_terminal
 from .auto_suggest import AutoSuggest, Suggestion
@@ -764,20 +766,24 @@ def auto_down(
 
     def delete_before_cursor(self, count: int = 1) -> str:
         """
-        Delete specified number of characters before cursor and return the
-        deleted text.
+        Delete specified number of grapheme clusters before cursor and return
+        the deleted text.
         """
         assert count >= 0
         deleted = ""
 
         if self.cursor_position > 0:
-            deleted = self.text[self.cursor_position - count : self.cursor_position]
-
-            new_text = (
-                self.text[: self.cursor_position - count]
-                + self.text[self.cursor_position :]
-            )
-            new_cursor_position = self.cursor_position - len(deleted)
+            # Find position after deleting `count` grapheme clusters.
+            # Loop is required since grapheme clusters have variable length.
+            pos = self.cursor_position
+            for _ in range(count):
+                if pos <= 0:
+                    break
+                pos = wcwidth.grapheme_boundary_before(self.text, pos)
+
+            deleted = self.text[pos : self.cursor_position]
+            new_text = self.text[:pos] + self.text[self.cursor_position :]
+            new_cursor_position = pos
 
             # Set new Document atomically.
             self.document = Document(new_text, new_cursor_position)
@@ -786,14 +792,19 @@ def delete_before_cursor(self, count: int = 1) -> str:
 
     def delete(self, count: int = 1) -> str:
         """
-        Delete specified number of characters and Return the deleted text.
+        Delete specified number of grapheme clusters and return the deleted text.
         """
         if self.cursor_position < len(self.text):
-            deleted = self.document.text_after_cursor[:count]
-            self.text = (
-                self.text[: self.cursor_position]
-                + self.text[self.cursor_position + len(deleted) :]
-            )
+            # Find position after `count` grapheme clusters.
+            text_after = self.text[self.cursor_position :]
+            pos = 0
+            for i, grapheme in enumerate(wcwidth.iter_graphemes(text_after)):
+                if i >= count:
+                    break
+                pos += len(grapheme)
+
+            deleted = text_after[:pos]
+            self.text = self.text[: self.cursor_position] + text_after[pos:]
             return deleted
         else:
             return ""

src/prompt_toolkit/document.py

@@ -10,6 +10,8 @@
 import weakref
 from typing import Callable, Dict, Iterable, List, NoReturn, Pattern, cast
 
+import wcwidth
+
 from .clipboard import ClipboardData
 from .filters import vi_mode
 from .selection import PasteMode, SelectionState, SelectionType
@@ -158,13 +160,49 @@ def selection(self) -> SelectionState | None:
 
     @property
     def current_char(self) -> str:
-        """Return character under cursor or an empty string."""
-        return self._get_char_relative_to_cursor(0) or ""
+        """
+        Return grapheme cluster at cursor position, or empty string at end.
+
+        Note: Returns a grapheme cluster which may contain multiple code points.
+        If cursor is inside a grapheme cluster (e.g., on a combining character),
+        returns the complete grapheme containing the cursor.
+        """
+        if self.cursor_position >= len(self.text):
+            return ""
+        grapheme_start = wcwidth.grapheme_boundary_before(
+            self.text, self.cursor_position + 1
+        )
+        for g in wcwidth.iter_graphemes(self.text[grapheme_start:]):
+            return g
+        return ""
 
     @property
     def char_before_cursor(self) -> str:
-        """Return character before the cursor or an empty string."""
-        return self._get_char_relative_to_cursor(-1) or ""
+        """
+        Return grapheme cluster before the cursor, or empty string at start.
+
+        Note: Returns a grapheme cluster which may contain multiple code points.
+        If cursor is inside a grapheme cluster (e.g., on a combining character),
+        returns the grapheme before the one containing the cursor.
+        """
+        if self.cursor_position == 0:
+            return ""
+
+        text = self.text
+        cursor = self.cursor_position
+
+        # Find reference point: cursor position or start of containing grapheme.
+        if cursor >= len(text):
+            reference = len(text)
+        else:
+            grapheme_start = wcwidth.grapheme_boundary_before(text, cursor + 1)
+            reference = grapheme_start if grapheme_start < cursor else cursor
+
+        if reference == 0:
+            return ""
+
+        prev_start = wcwidth.grapheme_boundary_before(text, reference)
+        return text[prev_start:reference]
 
     @property
     def text_before_cursor(self) -> str:
@@ -251,15 +289,6 @@ def leading_whitespace_in_current_line(self) -> str:
         length = len(current_line) - len(current_line.lstrip())
         return current_line[:length]
 
-    def _get_char_relative_to_cursor(self, offset: int = 0) -> str:
-        """
-        Return character relative to cursor position, or empty string
-        """
-        try:
-            return self.text[self.cursor_position + offset]
-        except IndexError:
-            return ""
-
     @property
     def on_first_line(self) -> bool:
         """
@@ -692,21 +721,44 @@ def find_previous_matching_line(
 
     def get_cursor_left_position(self, count: int = 1) -> int:
         """
-        Relative position for cursor left.
+        Relative position for cursor left (grapheme cluster aware).
         """
         if count < 0:
             return self.get_cursor_right_position(-count)
 
-        return -min(self.cursor_position_col, count)
+        line_before = self.current_line_before_cursor
+        if not line_before:
+            return 0
+
+        pos = len(line_before)
+        for _ in range(count):
+            if pos <= 0:
+                break
+            new_pos = wcwidth.grapheme_boundary_before(line_before, pos)
+            if new_pos == pos:
+                break
+            pos = new_pos
+
+        return pos - len(line_before)
 
     def get_cursor_right_position(self, count: int = 1) -> int:
         """
-        Relative position for cursor_right.
+        Relative position for cursor right (grapheme cluster aware).
         """
         if count < 0:
             return self.get_cursor_left_position(-count)
 
-        return min(count, len(self.current_line_after_cursor))
+        line_after = self.current_line_after_cursor
+        if not line_after:
+            return 0
+
+        pos = 0
+        for i, grapheme in enumerate(wcwidth.iter_graphemes(line_after)):
+            if i >= count:
+                break
+            pos += len(grapheme)
+
+        return pos
 
     def get_cursor_up_position(
         self, count: int = 1, preferred_column: int | None = None

src/prompt_toolkit/formatted_text/utils.py

@@ -9,7 +9,7 @@
 
 from typing import Iterable, cast
 
-from prompt_toolkit.utils import get_cwidth
+import wcwidth
 
 from .base import (
     AnyFormattedText,
@@ -48,17 +48,15 @@ def fragment_list_len(fragments: StyleAndTextTuples) -> int:
 def fragment_list_width(fragments: StyleAndTextTuples) -> int:
     """
     Return the character width of this text fragment list.
-    (Take double width characters into account.)
+    (Take double width characters and grapheme clusters into account.)
 
     :param fragments: List of ``(style_str, text)`` or
         ``(style_str, text, mouse_handler)`` tuples.
     """
-    ZeroWidthEscape = "[ZeroWidthEscape]"
     return sum(
-        get_cwidth(c)
+        wcwidth.width(item[1], control_codes="ignore")
         for item in fragments
-        for c in item[1]
-        if ZeroWidthEscape not in item[0]
+        if "[ZeroWidthEscape]" not in item[0]
     )
 
 

src/prompt_toolkit/layout/containers.py

@@ -10,6 +10,8 @@
 from functools import partial
 from typing import TYPE_CHECKING, Callable, Sequence, Union, cast
 
+import wcwidth
+
 from prompt_toolkit.application.current import get_app
 from prompt_toolkit.cache import SimpleCache
 from prompt_toolkit.data_structures import Point
@@ -2014,7 +2016,7 @@ def copy_line(
                     new_screen.zero_width_escapes[y + ypos][x + xpos] += text
                     continue
 
-                for c in text:
+                for c in wcwidth.iter_graphemes(text):
                     char = _CHAR_CACHE[c, style]
                     char_width = char.width
 
@@ -2052,26 +2054,7 @@ def copy_line(
                             for i in range(1, char_width):
                                 new_buffer_row[x + xpos + i] = empty_char
 
-                        # If this is a zero width characters, then it's
-                        # probably part of a decomposed unicode character.
-                        # See: https://en.wikipedia.org/wiki/Unicode_equivalence
-                        # Merge it in the previous cell.
-                        elif char_width == 0:
-                            # Handle all character widths. If the previous
-                            # character is a multiwidth character, then
-                            # merge it two positions back.
-                            for pw in [2, 1]:  # Previous character width.
-                                if (
-                                    x - pw >= 0
-                                    and new_buffer_row[x + xpos - pw].width == pw
-                                ):
-                                    prev_char = new_buffer_row[x + xpos - pw]
-                                    char2 = _CHAR_CACHE[
-                                        prev_char.char + c, prev_char.style
-                                    ]
-                                    new_buffer_row[x + xpos - pw] = char2
-
-                        # Keep track of write position for each character.
+                        # Keep track of write position for each grapheme.
                         current_rowcol_to_yx[lineno, col + skipped] = (
                             y + ypos,
                             x + xpos,

src/prompt_toolkit/layout/controls.py

@@ -8,6 +8,8 @@
 from abc import ABCMeta, abstractmethod
 from typing import TYPE_CHECKING, Callable, Hashable, Iterable, NamedTuple
 
+import wcwidth
+
 from prompt_toolkit.application.current import get_app
 from prompt_toolkit.buffer import Buffer
 from prompt_toolkit.cache import SimpleCache
@@ -674,29 +676,45 @@ def transform(
         ) -> _ProcessedLine:
             "Transform the fragments for a given line number."
 
-            # Get cursor position at this line.
-            def source_to_display(i: int) -> int:
-                """X position from the buffer to the x position in the
-                processed fragment list. By default, we start from the 'identity'
-                operation."""
-                return i
+            # Build code point to grapheme index mapping for cursor positioning.
+            line_text = fragment_list_to_text(fragments)
+            codepoint_to_grapheme: dict[int, int] = {}
+            grapheme_idx = 0
+            codepoint_idx = 0
+            for grapheme in wcwidth.iter_graphemes(line_text):
+                for _ in grapheme:
+                    codepoint_to_grapheme[codepoint_idx] = grapheme_idx
+                    codepoint_idx += 1
+                grapheme_idx += 1
+
+            def grapheme_source_to_display(i: int) -> int:
+                """Map code point index to grapheme index."""
+                if i >= codepoint_idx:
+                    return grapheme_idx + (i - codepoint_idx)
+                return codepoint_to_grapheme.get(i, grapheme_idx)
 
             transformation = merged_processor.apply_transformation(
                 TransformationInput(
                     self,
                     document,
                     lineno,
-                    source_to_display,
+                    grapheme_source_to_display,
                     fragments,
                     width,
                     height,
                     get_line,
                 )
             )
 
+            # Compose grapheme mapping with processor transformations.
+            proc_s2d = transformation.source_to_display
+
+            def final_source_to_display(i: int) -> int:
+                return proc_s2d(grapheme_source_to_display(i))
+
             return _ProcessedLine(
                 transformation.fragments,
-                transformation.source_to_display,
+                final_source_to_display,
                 transformation.display_to_source,
             )