From 696c81f48ef48630937d343b67fe8e4bf55c8998 Mon Sep 17 00:00:00 2001
From: Shamil Abdulaev <ashm.tech@proton.me>
Date: Sun, 13 Sep 2026 12:47:18 +0300
Subject: [PATCH] fix(tokenizer): correct offset of Non-UTF-8 syntax error

---
 Lib/test/test_exceptions.py                            | 10 ++++++++++
 .../2026-09-13-12-46-10.gh-issue-157378.Rg7Tq2.rst     |  3 +++
 Parser/tokenizer/helpers.c                             |  6 ++----
 3 files changed, 15 insertions(+), 4 deletions(-)
 create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-09-13-12-46-10.gh-issue-157378.Rg7Tq2.rst

diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py
index 0c02b38dd3c0a89..750e45bb1644686 100644
--- a/Lib/test/test_exceptions.py
+++ b/Lib/test/test_exceptions.py
@@ -250,6 +250,16 @@ def testSyntaxErrorRange(self):
                 self.assertEqual(cm.exception.offset, offset)
                 self.assertEqual(cm.exception.end_offset, end_offset)
 
+    def testSyntaxErrorNonUTF8Offset(self):
+        # gh-157378: the position was reported one column short for each
+        # multi-byte character preceding the invalid byte on the same line
+        check = self.check
+        check(b'X\x80', 1, 2, 1, 2)
+        check(b'\xc3\xa9X\x80', 1, 3, 1, 3)
+        check(b'\t\xc3\xa9X\x80', 1, 4, 1, 4)
+        check(b'a\xc3\xa9b\x80c', 1, 4, 1, 4)
+        check(b'a\n\xc3\xa9X\x80', 2, 3, 2, 3)
+
     def testSyntaxErrorOffset(self):
         check = self.check
         check('def fact(x):\n\treturn x!\n', 2, 10)
diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-13-12-46-10.gh-issue-157378.Rg7Tq2.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-13-12-46-10.gh-issue-157378.Rg7Tq2.rst
new file mode 100644
index 000000000000000..569642492992d3a
--- /dev/null
+++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-13-12-46-10.gh-issue-157378.Rg7Tq2.rst
@@ -0,0 +1,3 @@
+Fix ``SyntaxError.offset`` and ``SyntaxError.end_offset`` for the
+"Non-UTF-8 code starting with ..." error when a non-ASCII character precedes
+the invalid byte on the same line. Patch by Shamil Abdulaev.
diff --git a/Parser/tokenizer/helpers.c b/Parser/tokenizer/helpers.c
index c803b787d9dae68..d0ada5ac1131a71 100644
--- a/Parser/tokenizer/helpers.c
+++ b/Parser/tokenizer/helpers.c
@@ -329,17 +329,14 @@ _PyTokenizer_ensure_utf8(const char *line, struct tok_state *tok, int lineno)
     const char *badchar = NULL;
     const char *c;
     int length;
-    int col_offset = 0;
     const char *line_start = line;
     for (c = line; *c; c += length) {
         if (!(length = valid_utf8((const unsigned char *)c))) {
             badchar = c;
             break;
         }
-        col_offset++;
         if (*c == '\n') {
             lineno++;
-            col_offset = 0;
             line_start = c + 1;
         }
     }
@@ -348,7 +345,8 @@ _PyTokenizer_ensure_utf8(const char *line, struct tok_state *tok, int lineno)
         tok->line_start = _PyLexer_BufferOffset(tok, line_start);
         tok->cur = _PyLexer_BufferOffset(tok, badchar);
         _PyTokenizer_syntaxerror_known_range(tok,
-                col_offset + 1, col_offset + 1,
+                (int)(badchar - line_start) + 1,
+                (int)(badchar - line_start) + 1,
                 "Non-UTF-8 code starting with '\\x%.2x'"
                 "%s%V on line %i, "
                 "but no encoding declared; "