gh-161145: Use _Py_find_first_nonascii in the tokenizer

Skip sequence of ASCII at once rather than one char per loop reusing
faster code from unicodeobject. Can't use the checking from stringlib
or unicodeobject fully because those assume copying/writing to an output
buffer which slows down this case. Line numbers are only needed for the
error message, so count them there.  The function is exported because
test_peg_generator builds the tokenizer sources into an extension
module.

The check drops from about 0.9% of compile() time to about 0.1%
(perf record).  CPU time per compile(), PGO+LTO, 20 paired runs on one
pinned core:

| Workload | main | this | runs faster |
|---|---:|---:|---:|
| typing.py | 6.72 ms | 1.2% faster | 17/20 |
| all top-level Lib/*.py | 241 ms | 0.7% faster | 17/20 |
| synthetic CJK literals | 4.32 ms | 1.3% faster | 16/20 |

_PyTokenizer_ensure_utf8 instructions during compile(), callgrind:

| File | main | this |
|---|---:|---:|
| typing.py | 1,811,154 | 132,909 |
| _pydatetime.py | 1,246,691 | 91,566 |
| test_email/test_contentmanager.py | 487,525 | 43,273 |
| synthetic Cyrillic literals | 5,825,590 | 3,832,223 |
| synthetic CJK literals | 3,450,590 | 2,479,019 |

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Author: cmaloney

Include/internal/pycore_unicodeobject.h

@@ -234,6 +234,13 @@ extern PyObject* _PyUnicode_FromASCII(
     const char *buffer,
     Py_ssize_t size);
 
+/* Return the index of the first byte in [start, end) with the high bit set,
+   or end - start if there is none. */
+// Export for test_peg_generator.
+PyAPI_FUNC(Py_ssize_t) _Py_find_first_nonascii(
+    const unsigned char *start,
+    const unsigned char *end);
+
 /* Compute the maximum character of the substring unicode[start:end].
    Return 127 for an empty string. */
 extern Py_UCS4 _PyUnicode_FindMaxChar (

Objects/unicodeobject.c

@@ -5047,8 +5047,8 @@ load_unaligned(const unsigned char *p, size_t size)
  * significant bit set). If all characters in the range are ASCII, it returns
  * `end - start`.
  */
-static Py_ssize_t
-find_first_nonascii(const unsigned char *start, const unsigned char *end)
+Py_ssize_t
+_Py_find_first_nonascii(const unsigned char *start, const unsigned char *end)
 {
     // The search is done in `size_t` chunks.
     // The start and end might not be aligned at `size_t` boundaries,
@@ -5188,8 +5188,8 @@ ascii_decode(const char *start, const char *end, Py_UCS1 *dest)
         return p - start;
     }
 #endif
-    Py_ssize_t pos = find_first_nonascii((const unsigned char*)start,
-                                         (const unsigned char*)end);
+    Py_ssize_t pos = _Py_find_first_nonascii((const unsigned char*)start,
+                                             (const unsigned char*)end);
     memcpy(dest, start, pos);
     return pos;
 }
@@ -5348,7 +5348,8 @@ unicode_decode_utf8(const char *s, Py_ssize_t size,
     const char *starts = s;
     const char *end = s + size;
 
-    Py_ssize_t pos = find_first_nonascii((const unsigned char*)starts, (const unsigned char*)end);
+    Py_ssize_t pos = _Py_find_first_nonascii((const unsigned char*)starts,
+                                             (const unsigned char*)end);
     if (pos == size) {  // fast path: ASCII string.
         PyObject *u = PyUnicode_New(size, 127);
         if (u == NULL) {

Parser/tokenizer/helpers.c

@@ -3,6 +3,7 @@
 #include "pycore_runtime.h"       // _Py_ID()
 #include "pycore_token.h"
 #include "pycore_tuple.h"         // _PyTuple_FromPair
+#include "pycore_unicodeobject.h" // _Py_find_first_nonascii()
 
 #include "../lexer/state.h"
 
@@ -257,14 +258,14 @@ _PyTokenizer_parser_warn(struct tok_state *tok, PyObject *category, const char *
    the sequence if yes, 0 if not.  The special cases match
    those in stringlib/codecs.h:utf8_decode.
 */
-static int
-valid_utf8(const unsigned char* s)
+static Py_ssize_t
+valid_utf8(const unsigned char *s, const unsigned char *end)
 {
     int expected = 0;
     int length;
     if (*s < 0x80) {
-        /* single-byte code */
-        return 1;
+        /* as many ASCII chars as we can find in a row */
+        return _Py_find_first_nonascii(s, end);
     }
     else if (*s < 0xE0) {
         /* \xC2\x80-\xDF\xBF -- 0080-07FF */
@@ -321,19 +322,22 @@ _PyTokenizer_ensure_utf8(const char *line, struct tok_state *tok, int lineno)
 {
     const char *badchar = NULL;
     const char *c;
-    int length;
-    const char *line_start = line;
+    const unsigned char *end = (const unsigned char *)line + strlen(line);
+    Py_ssize_t length;
     for (c = line; *c; c += length) {
-        if (!(length = valid_utf8((const unsigned char *)c))) {
+        if (!(length = valid_utf8((const unsigned char *)c, end))) {
             badchar = c;
             break;
         }
-        if (*c == '\n') {
-            lineno++;
-            line_start = c + 1;
-        }
     }
     if (badchar) {
+        const char *line_start = line;
+        for (c = line; c < badchar; c++) {
+            if (*c == '\n') {
+                lineno++;
+                line_start = c + 1;
+            }
+        }
         _PyTokenizer_syntaxerror_at(
                 tok, line_start, badchar - line_start + 1, lineno,
                 -1, -1,