fix(tokenizer): correct offset of Non-UTF-8 syntax error

Author: ashm-dev

Lib/test/test_exceptions.py

@@ -250,6 +250,16 @@ def testSyntaxErrorRange(self):
                 self.assertEqual(cm.exception.offset, offset)
                 self.assertEqual(cm.exception.end_offset, end_offset)
 
+    def testSyntaxErrorNonUTF8Offset(self):
+        # gh-157378: the position was reported one column short for each
+        # multi-byte character preceding the invalid byte on the same line
+        check = self.check
+        check(b'X\x80', 1, 2, 1, 2)
+        check(b'\xc3\xa9X\x80', 1, 3, 1, 3)
+        check(b'\t\xc3\xa9X\x80', 1, 4, 1, 4)
+        check(b'a\xc3\xa9b\x80c', 1, 4, 1, 4)
+        check(b'a\n\xc3\xa9X\x80', 2, 3, 2, 3)
+
     def testSyntaxErrorOffset(self):
         check = self.check
         check('def fact(x):\n\treturn x!\n', 2, 10)

Misc/NEWS.d/next/Core_and_Builtins/2026-09-13-12-46-10.gh-issue-157378.Rg7Tq2.rst

@@ -0,0 +1,3 @@
+Fix ``SyntaxError.offset`` and ``SyntaxError.end_offset`` for the
+"Non-UTF-8 code starting with ..." error when a non-ASCII character precedes
+the invalid byte on the same line. Patch by Shamil Abdulaev.

Parser/tokenizer/helpers.c

@@ -329,17 +329,14 @@ _PyTokenizer_ensure_utf8(const char *line, struct tok_state *tok, int lineno)
     const char *badchar = NULL;
     const char *c;
     int length;
-    int col_offset = 0;
     const char *line_start = line;
     for (c = line; *c; c += length) {
         if (!(length = valid_utf8((const unsigned char *)c))) {
             badchar = c;
             break;
         }
-        col_offset++;
         if (*c == '\n') {
             lineno++;
-            col_offset = 0;
             line_start = c + 1;
         }
     }
@@ -348,7 +345,8 @@ _PyTokenizer_ensure_utf8(const char *line, struct tok_state *tok, int lineno)
         tok->line_start = _PyLexer_BufferOffset(tok, line_start);
         tok->cur = _PyLexer_BufferOffset(tok, badchar);
         _PyTokenizer_syntaxerror_known_range(tok,
-                col_offset + 1, col_offset + 1,
+                (int)(badchar - line_start) + 1,
+                (int)(badchar - line_start) + 1,
                 "Non-UTF-8 code starting with '\\x%.2x'"
                 "%s%V on line %i, "
                 "but no encoding declared; "