Preserve UTF-8 diagnostic columns after tokenizer API cutover

Author: pablogsal

Lib/test/test_source_encoding.py

@@ -3,8 +3,8 @@
 import unittest
 from test import support
 from test.support import script_helper
-from test.support.os_helper import TESTFN, unlink, rmtree
-from test.support.import_helper import unload
+from test.support.os_helper import TESTFN, TESTFN_ASCII, unlink, rmtree
+from test.support.import_helper import import_module, unload
 import importlib
 import os
 import sys
@@ -83,12 +83,30 @@ def test_truncated_utf8_at_eof(self):
                 self.assertRaises(SyntaxError, compile, seq, '<test>', 'exec')
 
     def test_invalid_utf8_offset_after_non_ascii(self):
+        for name in ('é', 'éé', '𝒜'):
+            with self.subTest(name=name):
+                source = ('x = ' + name).encode() + b'\xff\n'
+                with self.assertRaises(SyntaxError) as caught:
+                    compile(source, '<test>', 'exec')
+                error = caught.exception
+                self.assertEqual(
+                    (error.lineno, error.offset, error.end_lineno, error.end_offset),
+                    (1, 5 + len(name), 1, 5 + len(name)),
+                )
+
+    @support.cpython_only
+    def test_invalid_utf8_file_offset_after_non_ascii(self):
+        _testcapi = import_module('_testcapi')
+        self.addCleanup(unlink, TESTFN_ASCII)
+        with open(TESTFN_ASCII, 'wb') as f:
+            f.write(b'\nx = \xc3\xa9\xc3\xa9\xff\n')
         with self.assertRaises(SyntaxError) as caught:
-            compile(b"x = \xc3\xa9\xff\n", "<test>", "exec")
+            _testcapi.run_file(
+                os.fsencode(TESTFN_ASCII), _testcapi.Py_file_input, {})
         error = caught.exception
         self.assertEqual(
             (error.lineno, error.offset, error.end_lineno, error.end_offset),
-            (1, 6, 1, 6),
+            (2, 7, 2, 7),
         )
 
     def test_long_bom_conflict_message_is_not_truncated(self):

Parser/tokenizer/helpers.c

@@ -322,24 +322,22 @@ _PyTokenizer_ensure_utf8(const char *line, struct tok_state *tok, int lineno)
     const char *badchar = NULL;
     const char *c;
     int length;
-    int col_offset = 0;
     const char *line_start = line;
     for (c = line; *c; c += length) {
         if (!(length = valid_utf8((const unsigned char *)c))) {
             badchar = c;
             break;
         }
-        col_offset++;
         if (*c == '\n') {
             lineno++;
-            col_offset = 0;
             line_start = c + 1;
         }
     }
     if (badchar) {
+        int col_offset = (int)(badchar - line_start) + 1;
         _PyTokenizer_syntaxerror_at(
                 tok, line_start, badchar - line_start, lineno,
-                col_offset + 1, col_offset + 1,
+                col_offset, col_offset,
                 "Non-UTF-8 code starting with '\\x%.2x'"
                 "%s%V on line %i, "
                 "but no encoding declared; "

Parser/tokenizer/source.h

@@ -37,7 +37,7 @@ PyAPI_FUNC(_PyTok_Off) _PyTok_SourceAppendLine(
 /* Return borrowed bytes excluding '\n', writing the byte length to *len.
    Line numbers are 1-based and clamp to the first or final line; a trailing
    '\n' adds an empty final line. The view need not be NUL-terminated.
-   This does not set an exception. Append and clear invalidate the view. */
+   This does not set an exception. Append, discard, and clear invalidate the view. */
 PyAPI_FUNC(const char *) _PyTok_SourceLineView(
     const _PyTok_SourceText *source, Py_ssize_t lineno, Py_ssize_t *len);
 /* Return false for invalid line numbers and the virtual EOF line. */