gh-153569: move tokenizer input state and relocation into the reader

Author: pablogsal

Lib/test/test_repl.py

@@ -184,9 +184,8 @@ def read_until(marker, start=0):
 
     @cpython_only
     def test_lexer_buffer_realloc_with_null_start(self):
-        # gh-144759: NULL pointer arithmetic in the lexer when start and
-        # multi_line_start are NULL (uninitialized in tok_mode_stack[0])
-        # and the lexer buffer is reallocated while parsing long input.
+        # gh-144759: NULL pointer arithmetic when the lexer buffer grows
+        # while parsing long input.
         long_value = "a" * 2000
         user_input = dedent(f"""\
         x = f'{{{long_value!r}}}'

Makefile.pre.in

@@ -394,7 +394,6 @@ PEGEN_OBJS=		\
 		Parser/peg_api.o
 
 TOKENIZER_OBJS=		\
-		Parser/lexer/buffer.o \
 		Parser/lexer/lexer.o \
 		Parser/lexer/number.o \
 		Parser/lexer/state.o \
@@ -411,7 +410,6 @@ PEGEN_HEADERS= \
 		$(srcdir)/Parser/string_parser.h
 
 TOKENIZER_HEADERS= \
-		Parser/lexer/buffer.h \
 		Parser/lexer/lexer.h \
 		Parser/lexer/lexer_internal.h \
 		Parser/lexer/state.h \

PCbuild/_freeze_module.vcxproj

@@ -181,7 +181,6 @@
     <ClCompile Include="..\Parser\action_helpers.c" />
     <ClCompile Include="..\Parser\string_parser.c" />
     <ClCompile Include="..\Parser\token.c" />
-    <ClCompile Include="..\Parser\lexer\buffer.c" />
     <ClCompile Include="..\Parser\lexer\state.c" />
     <ClCompile Include="..\Parser\lexer\lexer.c" />
     <ClCompile Include="..\Parser\lexer\number.c" />

PCbuild/_freeze_module.vcxproj.filters

@@ -469,9 +469,6 @@
     <ClCompile Include="..\Parser\lexer\string.c">
       <Filter>Source Files</Filter>
     </ClCompile>
-    <ClCompile Include="..\Parser\lexer\buffer.c">
-      <Filter>Source Files</Filter>
-    </ClCompile>
     <ClCompile Include="..\Parser\lexer\state.c">
       <Filter>Source Files</Filter>
     </ClCompile>

PCbuild/pythoncore.vcxproj

@@ -423,10 +423,9 @@
     <ClInclude Include="..\Parser\lexer\state.h" />
     <ClInclude Include="..\Parser\lexer\lexer.h" />
     <ClInclude Include="..\Parser\lexer\lexer_internal.h" />
-    <ClInclude Include="..\Parser\lexer\buffer.h" />
-    <ClInclude Include="..\Parser\tokenizer\cursor.h" />
     <ClInclude Include="..\Parser\tokenizer\reader.h" />
     <ClInclude Include="..\Parser\tokenizer\reader_internal.h" />
+    <ClInclude Include="..\Parser\tokenizer\cursor.h" />
     <ClInclude Include="..\Parser\tokenizer\source.h" />
     <ClInclude Include="..\Parser\tokenizer\helpers.h" />
     <ClInclude Include="..\Parser\tokenizer\tokenizer.h" />
@@ -593,7 +592,6 @@
     <ClCompile Include="..\Parser\lexer\lexer.c" />
     <ClCompile Include="..\Parser\lexer\number.c" />
     <ClCompile Include="..\Parser\lexer\string.c" />
-    <ClCompile Include="..\Parser\lexer\buffer.c" />
     <ClCompile Include="..\Parser\tokenizer\cursor.c" />
     <ClCompile Include="..\Parser\tokenizer\source.c" />
     <ClCompile Include="..\Parser\tokenizer\decoder.c" />

PCbuild/pythoncore.vcxproj.filters

@@ -330,18 +330,15 @@
     <ClInclude Include="..\Parser\lexer\lexer_internal.h">
       <Filter>Parser</Filter>
     </ClInclude>
-    <ClInclude Include="..\Parser\lexer\buffer.h">
-      <Filter>Parser</Filter>
-    </ClInclude>
-    <ClInclude Include="..\Parser\tokenizer\cursor.h">
-      <Filter>Parser</Filter>
-    </ClInclude>
     <ClInclude Include="..\Parser\tokenizer\reader.h">
       <Filter>Parser</Filter>
     </ClInclude>
     <ClInclude Include="..\Parser\tokenizer\reader_internal.h">
       <Filter>Parser</Filter>
     </ClInclude>
+    <ClInclude Include="..\Parser\tokenizer\cursor.h">
+      <Filter>Parser</Filter>
+    </ClInclude>
     <ClInclude Include="..\Parser\tokenizer\source.h">
       <Filter>Parser</Filter>
     </ClInclude>
@@ -1361,9 +1358,6 @@
     <ClCompile Include="..\Parser\lexer\state.c">
       <Filter>Parser</Filter>
     </ClCompile>
-    <ClCompile Include="..\Parser\lexer\buffer.c">
-      <Filter>Parser</Filter>
-    </ClCompile>
     <ClCompile Include="..\Parser\tokenizer\cursor.c">
       <Filter>Parser</Filter>
     </ClCompile>

Parser/lexer/buffer.c

@@ -207,15 +207,16 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token
         }
         tok_backup(tok, c);
         if (c == '#' || c == '\n' || c == '\r') {
+            int interactive = _PyTok_ReaderIsInteractive(tok);
             /* Lines with only whitespace and/or comments
                shouldn't affect the indentation and are
                not passed to the parser as NEWLINE tokens,
                except *totally* empty lines in interactive
                mode, which signal the end of a command group. */
-            if (col == 0 && c == '\n' && tok->prompt != NULL) {
+            if (col == 0 && c == '\n' && interactive) {
                 blankline = 0; /* Let it through */
             }
-            else if (tok->prompt != NULL && tok->lineno == 1) {
+            else if (interactive && tok->lineno == 1) {
                 /* In interactive mode, if the first line contains
                    only spaces and/or a comment, let it through. */
                 blankline = 0;

Parser/lexer/buffer.h

@@ -1,60 +1,11 @@
 #include "Python.h"
-#include "pycore_pystate.h"
 #include "pycore_token.h"
 #include "errcode.h"
 
 #include "state.h"
 #include "../tokenizer/helpers.h"
 #include "../tokenizer/reader.h"
 
-/* Create and initialize a new tok_state structure */
-struct tok_state *
-_PyTokenizer_tok_new(void)
-{
-    struct tok_state *tok = (struct tok_state *)PyMem_Calloc(
-                                            1,
-                                            sizeof(struct tok_state));
-    if (tok == NULL) {
-        PyErr_NoMemory();
-        return NULL;
-    }
-
-    tok->buf = tok->cur = tok->inp = NULL;
-    tok->fp_interactive = 0;
-    tok->interactive_src_start = NULL;
-    tok->interactive_src_end = NULL;
-    tok->start = NULL;
-    tok->done = E_OK;
-    tok->fp = NULL;
-    tok->indent = 0;
-    tok->indstack[0] = 0;
-    tok->atbol = 1;
-    tok->pendin = 0;
-    tok->prompt = NULL;
-    tok->lineno = 0;
-    tok->start_loc = (_PyTok_Loc){-1, -1};
-    tok->level = 0;
-    tok->altindstack[0] = 0;
-    tok->encoding = NULL;
-    tok->filename = NULL;
-    tok->module = NULL;
-    tok->type_comments = 0;
-    tok->interactive_underflow = IUNDERFLOW_NORMAL;
-    tok->str = NULL;
-    tok->report_warnings = 1;
-    tok->tok_extra_tokens = 0;
-    tok->comment_newline = 0;
-    tok->implicit_newline = 0;
-    _PyTok_SourceInit(&tok->source);
-    tok->reader = NULL;
-    tok->ftstring_stack = tok->ftstring_stack_inline;
-    tok->ftstring_capacity = FTSTRING_STACK_INLINE_CAPACITY;
-#ifdef Py_DEBUG
-    tok->debug = _Py_GetConfig()->parser_debug;
-#endif
-    return tok;
-}
-
 ftstring_state *
 _PyLexer_PushFTString(struct tok_state *tok)
 {