diff --git a/Lib/test/test_capi/test_tokenizer.py b/Lib/test/test_capi/test_tokenizer.py index eb04f6c0136022d..57d0a3c2f2e99e8 100644 --- a/Lib/test/test_capi/test_tokenizer.py +++ b/Lib/test/test_capi/test_tokenizer.py @@ -12,9 +12,6 @@ def test_source(self): def test_source_discard(self): _testinternalcapi.test_tokenizer_source_discard() - def test_cursor(self): - _testinternalcapi.test_tokenizer_cursor() - if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_codeop.py b/Lib/test/test_codeop.py index d57452602ce5574..c75c0f8627bf084 100644 --- a/Lib/test/test_codeop.py +++ b/Lib/test/test_codeop.py @@ -113,6 +113,17 @@ def test_valid(self, compiler): av("def f():\n pass\n#foo\n") av("@a.b.c\ndef f():\n pass\n") + @subTests('symbol', ('single', 'exec')) + @subTests('prefix', ('', 'f', 't')) + def test_incomplete_string_diagnostics(self, symbol, prefix): + opening = f' á = {prefix}"""first\n' + source = 'if True:\n' + opening + 'second' + with self.assertRaises(_IncompleteInputError) as cm: + Compile()(source, '', symbol) + text = opening + 'second' + ('\n' if symbol == 'exec' else '') + self.assertEqual(cm.exception.args, ( + 'incomplete input', ('', 2, 9, text, 2, -1))) + @subTests('compiler', COMPILERS) def test_incomplete(self, compiler): ai = functools.partial(self.assertIncomplete, compiler=compiler) diff --git a/Lib/test/test_repl.py b/Lib/test/test_repl.py index 372c110783bce7a..e6eb7df16d23214 100644 --- a/Lib/test/test_repl.py +++ b/Lib/test/test_repl.py @@ -184,9 +184,8 @@ def read_until(marker, start=0): @cpython_only def test_lexer_buffer_realloc_with_null_start(self): - # gh-144759: NULL pointer arithmetic in the lexer when start and - # multi_line_start are NULL (uninitialized in tok_mode_stack[0]) - # and the lexer buffer is reallocated while parsing long input. + # gh-144759: NULL pointer arithmetic when the lexer buffer grows + # while parsing long input. long_value = "a" * 2000 user_input = dedent(f"""\ x = f'{{{long_value!r}}}' diff --git a/Lib/test/test_source_encoding.py b/Lib/test/test_source_encoding.py index 862a20a058be75a..ec98e609c4e98f9 100644 --- a/Lib/test/test_source_encoding.py +++ b/Lib/test/test_source_encoding.py @@ -3,8 +3,8 @@ import unittest from test import support from test.support import script_helper -from test.support.os_helper import TESTFN, unlink, rmtree -from test.support.import_helper import unload +from test.support.os_helper import TESTFN, TESTFN_ASCII, unlink, rmtree +from test.support.import_helper import import_module, unload import importlib import os import sys @@ -83,12 +83,30 @@ def test_truncated_utf8_at_eof(self): self.assertRaises(SyntaxError, compile, seq, '', 'exec') def test_invalid_utf8_offset_after_non_ascii(self): + for name in ('é', 'éé', '𝒜'): + with self.subTest(name=name): + source = ('x = ' + name).encode() + b'\xff\n' + with self.assertRaises(SyntaxError) as caught: + compile(source, '', 'exec') + error = caught.exception + self.assertEqual( + (error.lineno, error.offset, error.end_lineno, error.end_offset), + (1, 5 + len(name), 1, 5 + len(name)), + ) + + @support.cpython_only + def test_invalid_utf8_file_offset_after_non_ascii(self): + _testcapi = import_module('_testcapi') + self.addCleanup(unlink, TESTFN_ASCII) + with open(TESTFN_ASCII, 'wb') as f: + f.write(b'\nx = \xc3\xa9\xc3\xa9\xff\n') with self.assertRaises(SyntaxError) as caught: - compile(b"x = \xc3\xa9\xff\n", "", "exec") + _testcapi.run_file( + os.fsencode(TESTFN_ASCII), _testcapi.Py_file_input, {}) error = caught.exception self.assertEqual( (error.lineno, error.offset, error.end_lineno, error.end_offset), - (1, 6, 1, 6), + (2, 7, 2, 7), ) def test_long_bom_conflict_message_is_not_truncated(self): diff --git a/Lib/test/test_tstring.py b/Lib/test/test_tstring.py index 854860b5ea43065..332e048f18d3ff6 100644 --- a/Lib/test/test_tstring.py +++ b/Lib/test/test_tstring.py @@ -215,6 +215,8 @@ def test_nested_templates(self): def test_syntax_errors(self): for case, err in ( + ('t"""{(\n1\n)}\ntail', "unterminated triple-quoted t-string literal"), + ('f"""{(\n1\n)}\ntail', "unterminated triple-quoted f-string literal"), ("t'", "unterminated t-string literal"), ("t'''", "unterminated triple-quoted t-string literal"), ("t''''", "unterminated triple-quoted t-string literal"), diff --git a/Makefile.pre.in b/Makefile.pre.in index 166087f32dff187..445c881d775aac7 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -395,10 +395,10 @@ PEGEN_OBJS= \ TOKENIZER_OBJS= \ Parser/lexer/lexer.o \ + Parser/lexer/layout.o \ Parser/lexer/number.o \ Parser/lexer/state.o \ Parser/lexer/string.o \ - Parser/tokenizer/cursor.o \ Parser/tokenizer/decoder.o \ Parser/tokenizer/api.o \ Parser/tokenizer/reader.o \ @@ -411,10 +411,8 @@ PEGEN_HEADERS= \ $(srcdir)/Parser/string_parser.h TOKENIZER_HEADERS= \ - Parser/lexer/lexer.h \ Parser/lexer/lexer_internal.h \ Parser/lexer/state.h \ - Parser/tokenizer/cursor.h \ Parser/tokenizer/reader.h \ Parser/tokenizer/reader_internal.h \ Parser/tokenizer/source.h \ @@ -3463,7 +3461,7 @@ MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo. MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_openssl_mem.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/parts.h $(srcdir)/Modules/_testcapi/util.h MODULE__TESTLIMITEDCAPI_DEPS=$(srcdir)/Modules/_testlimitedcapi/testcapi_long.h $(srcdir)/Modules/_testlimitedcapi/parts.h $(srcdir)/Modules/_testlimitedcapi/util.h -MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h $(srcdir)/Parser/tokenizer/cursor.h $(srcdir)/Parser/tokenizer/source.h $(srcdir)/Parser/tokenizer/types.h $(srcdir)/Python/ceval.h $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h +MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h $(srcdir)/Parser/tokenizer/source.h $(srcdir)/Parser/tokenizer/types.h $(srcdir)/Python/ceval.h $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h MODULE__SQLITE3_DEPS=$(srcdir)/Modules/_sqlite/connection.h $(srcdir)/Modules/_sqlite/cursor.h $(srcdir)/Modules/_sqlite/microprotocols.h $(srcdir)/Modules/_sqlite/module.h $(srcdir)/Modules/_sqlite/prepare_protocol.h $(srcdir)/Modules/_sqlite/row.h $(srcdir)/Modules/_sqlite/util.h MODULE__ZSTD_DEPS=$(srcdir)/Modules/_zstd/_zstdmodule.h $(srcdir)/Modules/_zstd/buffer.h $(srcdir)/Modules/_zstd/zstddict.h diff --git a/Modules/_testinternalcapi/tokenizer.c b/Modules/_testinternalcapi/tokenizer.c index df481cb832a4363..1f89c12f223c7d9 100644 --- a/Modules/_testinternalcapi/tokenizer.c +++ b/Modules/_testinternalcapi/tokenizer.c @@ -1,6 +1,6 @@ #include "parts.h" -#include "../../Parser/tokenizer/cursor.h" +#include "../../Parser/tokenizer/source.h" static int check(int condition, const char *message) @@ -24,13 +24,14 @@ check_system_error(int failed, const char *message) } static int -same_cursor(const _PyTok_Cursor *left, const _PyTok_Cursor *right) +check_line_view(const _PyTok_SourceText *source, Py_ssize_t lineno, + const char *expected) { - return left->source == right->source && - left->pos == right->pos && - left->line_start == right->line_start && - left->line_end == right->line_end && - left->lineno == right->lineno; + Py_ssize_t len; + const char *line = _PyTok_SourceLineView(source, lineno, &len); + return check(len == (Py_ssize_t)strlen(expected) && + memcmp(line, expected, len) == 0, + "wrong source line view"); } static PyObject * @@ -40,300 +41,54 @@ test_tokenizer_source(PyObject *Py_UNUSED(module), _PyTok_SourceText source; _PyTok_SourceInit(&source); - _PyTok_Loc loc; - _PyTok_Line line; - if (check(_PyTok_SourceLocation( - &source, 0, _PYTOK_AFFINITY_RIGHT, &loc) == 0, - "cannot locate empty source") < 0 || - check(loc.lineno == 1 && loc.byte_col == 0, - "wrong empty source location") < 0 || - check(_PyTok_SourceLine(&source, 1, &line) == 0, - "cannot find empty source line") < 0 || - check(line.start == 0 && line.end == 0, - "wrong empty source line") < 0 || - check_system_error( - _PyTok_SourceAppendLine(&source, "", 0, 0) < 0, - "accepted empty source line") < 0 || + if (check_line_view(&source, 1, "") < 0) { + goto error; + } + + if (check_system_error( + _PyTok_SourceAppendLine(&source, "", 0, 0) < 0, + "accepted empty source line") < 0 || check_system_error( _PyTok_SourceAppendLine(&source, "a\nb\n", 4, 0) < 0, "accepted multiple source lines") < 0 || check_system_error( _PyTok_SourceAppendLine(&source, "a", 1, 1) < 0, - "accepted missing implicit newline") < 0) { - goto error; - } - - if (check(_PyTok_SourceAppendLine(&source, "alpha\n", 6, 0) == 0, - "wrong first source offset") < 0 || + "accepted missing implicit newline") < 0 || + check(_PyTok_SourceAppendLine( + &source, "alpha\n", 6, 0) == 0, + "wrong first source offset") < 0 || check(_PyTok_SourceAppendLine( &source, "\xce\xb2\n", 3, 1) == 6, "wrong second source offset") < 0 || - check(_PyTok_SourceAppendLine( - &source, "nul\0x\n", 6, 0) == 9, - "wrong third source offset") < 0) { - goto error; - } - - int marker_line = 257; - int final_line = 300; - _PyTok_Off marker_start = -1; - for (int lineno = 4; lineno <= final_line; lineno++) { - const char *text = lineno == marker_line ? "marker\n" : "x\n"; - Py_ssize_t len = (Py_ssize_t)strlen(text); - _PyTok_Off start = _PyTok_SourceAppendLine( - &source, text, len, lineno == final_line); - if (start < 0) { - goto error; - } - if (lineno == marker_line) { - marker_start = start; - } - } - - if (check(source.nlines == final_line, "wrong source line count") < 0 || - check(_PyTok_SourceLine(&source, marker_line, &line) == 0, - "cannot find late source line") < 0 || - check(line.start == marker_start && - line.end == marker_start + 7, - "wrong late source line") < 0 || - check(!line.implicit_newline && !line.contains_nul, - "wrong late source flags") < 0 || - check(_PyTok_SourceLine(&source, 2, &line) == 0, - "cannot find second source line") < 0 || - check(line.start == 6 && line.end == 9 && - line.implicit_newline && !line.contains_nul, - "wrong second source line") < 0 || check(!_PyTok_SourceLineIsImplicit(&source, 1) && _PyTok_SourceLineIsImplicit(&source, 2), - "wrong early implicit newline flags") < 0 || - check(_PyTok_SourceLine(&source, 3, &line) == 0, - "cannot find third source line") < 0 || - check(line.contains_nul, "missing null byte flag") < 0 || - check(_PyTok_SourceLine(&source, final_line, &line) == 0, - "cannot find final source line") < 0 || - check(line.implicit_newline && - _PyTok_SourceLineIsImplicit(&source, final_line), - "missing late implicit newline flag") < 0) { + "wrong implicit newline flags") < 0) { goto error; } - Py_ssize_t view_len; - const char *view = _PyTok_SourceSpanView( - &source, _PyTok_SpanFromBounds(6, 8), &view_len); - if (check(view != NULL && view_len == 2 && - memcmp(view, "\xce\xb2", 2) == 0, - "wrong source span view") < 0 || - check(_PyTok_SourceLocation( - &source, marker_start, - _PYTOK_AFFINITY_LEFT, &loc) == 0, - "cannot locate left line boundary") < 0 || - check(loc.lineno == marker_line - 1 && loc.byte_col == 2, - "wrong left boundary location") < 0 || - check(_PyTok_SourceLocation( - &source, marker_start, - _PYTOK_AFFINITY_RIGHT, &loc) == 0, - "cannot locate right line boundary") < 0 || - check(loc.lineno == marker_line && loc.byte_col == 0, - "wrong right boundary location") < 0 || - check(_PyTok_SourceLocation( - &source, marker_start + 1, - _PYTOK_AFFINITY_RIGHT, &loc) == 0, - "cannot locate late source byte") < 0 || - check(loc.lineno == marker_line && loc.byte_col == 1, - "wrong late source location") < 0) { + if (check_line_view(&source, PY_SSIZE_T_MIN, "alpha") < 0 || + check_line_view(&source, 1, "alpha") < 0 || + check_line_view(&source, 2, "\xce\xb2") < 0 || + check_line_view(&source, 3, "") < 0 || + check_line_view(&source, PY_SSIZE_T_MAX, "") < 0) { goto error; } - if (check(_PyTok_SourceLocation( - &source, source.len, _PYTOK_AFFINITY_LEFT, &loc) == 0, - "cannot locate left EOF") < 0 || - check(loc.lineno == final_line && loc.byte_col == 2, - "wrong left EOF location") < 0 || - check(_PyTok_SourceLocation( - &source, source.len, - _PYTOK_AFFINITY_RIGHT, &loc) == 0, - "cannot locate right EOF") < 0 || - check(loc.lineno == final_line + 1 && loc.byte_col == 0, - "wrong right EOF location") < 0 || - check(_PyTok_SourceLine(&source, final_line + 1, &line) == 0, - "cannot find virtual EOF line") < 0 || - check(line.start == source.len && line.end == source.len, - "wrong virtual EOF line") < 0 || - check(!_PyTok_SourceLineIsImplicit(&source, 0) && - !_PyTok_SourceLineIsImplicit( - &source, final_line + 1), - "virtual or invalid line is implicit") < 0) { - goto error; - } - - view = _PyTok_SourceSpanView( - &source, _PyTok_SpanFromBounds(0, source.len + 1), &view_len); - if (check_system_error(view == NULL, "accepted invalid source span") < 0 || - check_system_error( - _PyTok_SourceLocation( - &source, source.len + 1, - _PYTOK_AFFINITY_RIGHT, &loc) < 0, - "accepted invalid source offset") < 0 || - check_system_error( - _PyTok_SourceLine(&source, final_line + 2, &line) < 0, - "accepted invalid source line") < 0) { + if (check(source.len == 9 && + memcmp(source.bytes, "alpha\n\xce\xb2\n", 10) == 0, + "wrong source contents") < 0) { goto error; } _PyTok_SourceClear(&source); - _PyTok_SourceInit(&source); if (_PyTok_SourceAppendLine(&source, "tail", 4, 0) < 0 || check_system_error( _PyTok_SourceAppendLine(&source, "x\n", 2, 0) < 0, - "appended after unterminated source line") < 0 || - check(_PyTok_SourceLocation( - &source, source.len, - _PYTOK_AFFINITY_RIGHT, &loc) == 0, - "cannot locate unterminated EOF") < 0 || - check(loc.lineno == 1 && loc.byte_col == 4, - "wrong unterminated EOF location") < 0) { - goto error; - } - - _PyTok_SourceDiscard(&source); - if (check(_PyTok_SourceAppendLine(&source, "a\n", 2, 0) == 4, - "wrong retained source offset") < 0 || - _PyTok_SourceLine(&source, 1, &line) < 0 || - check(line.start == 4 && line.end == 6, - "wrong retained source line") < 0 || - _PyTok_SourceLocation( - &source, 4, _PYTOK_AFFINITY_LEFT, &loc) < 0 || - check(loc.lineno == 1 && loc.byte_col == 0, - "wrong retained source location") < 0) { - goto error; - } - view = _PyTok_SourceSpanView( - &source, _PyTok_SpanFromBounds(4, 5), &view_len); - if (check(view != NULL && view_len == 1 && view[0] == 'a', - "wrong retained source span") < 0 || - check_system_error(_PyTok_SourceSpanView( - &source, _PyTok_SpanFromBounds(0, 1), &view_len) == NULL, - "accepted discarded source span") < 0) { - goto error; - } - - _PyTok_SourceClear(&source); - Py_RETURN_NONE; - -error: - _PyTok_SourceClear(&source); - return NULL; -} - -static PyObject * -test_tokenizer_cursor(PyObject *Py_UNUSED(module), - PyObject *Py_UNUSED(args)) -{ - _PyTok_SourceText source; - _PyTok_SourceInit(&source); - if (_PyTok_SourceAppendLine(&source, "ab\n", 3, 0) < 0 || - _PyTok_SourceAppendLine(&source, "cd\n", 3, 0) < 0) { - goto error; - } - - _PyTok_Cursor cursor; - _PyTok_CursorInit(&cursor, &source); - if (_PyTok_CursorSetOffset(&cursor, source.len) < 0 || - check(cursor.lineno == 3 && cursor.pos == source.len, - "wrong cursor at virtual EOF") < 0 || - _PyTok_CursorSetLine(&cursor, 1) < 0) { - goto error; - } - - char large[BUFSIZ + 1]; - memset(large, 'z', sizeof(large)); - large[sizeof(large) - 1] = '\n'; - if (_PyTok_SourceAppendLine(&source, large, sizeof(large), 0) < 0) { - goto error; - } - - if (check(_PyTok_CursorPeek(&cursor, 0) == 'a', - "wrong cursor peek after relocation") < 0 || - check(_PyTok_CursorPeek(&cursor, 1) == 'b', - "wrong distant cursor peek") < 0 || - check(_PyTok_CursorAdvance(&cursor) == 'a', - "wrong first cursor byte") < 0 || - check(_PyTok_CursorAdvance(&cursor) == 'b', - "wrong second cursor byte") < 0 || - check(_PyTok_CursorAdvance(&cursor) == '\n', - "wrong final cursor byte") < 0 || - check(_PyTok_CursorAdvance(&cursor) == EOF, - "cursor advanced past line") < 0 || - check(_PyTok_CursorSetOffset(&cursor, 2) == 0, - "cannot seek cursor offset") < 0 || - check(_PyTok_CursorAdvance(&cursor) == '\n', - "wrong cursor byte after seek") < 0 || - check(_PyTok_CursorSetOffset(&cursor, 3) == 0, - "cannot seek line boundary") < 0 || - check(cursor.lineno == 2 && cursor.line_start == 3 && - _PyTok_CursorAdvance(&cursor) == 'c', - "wrong cursor at line boundary") < 0 || - check(_PyTok_CursorSetLine(&cursor, 3) == 0, - "cannot advance cursor to final line") < 0 || - check(cursor.line_start == 6 && - _PyTok_CursorAdvance(&cursor) == 'z', - "wrong cursor byte on final line") < 0) { - goto error; - } - - _PyTok_Cursor saved = cursor; - if (check_system_error( - _PyTok_CursorSetOffset(&cursor, source.len + 1) < 0, - "accepted invalid cursor offset") < 0 || - check(same_cursor(&cursor, &saved), - "invalid offset changed cursor") < 0 || - check_system_error( - _PyTok_CursorSetLine(&cursor, source.nlines + 2) < 0, - "accepted invalid cursor line") < 0 || - check(same_cursor(&cursor, &saved), - "invalid line changed cursor") < 0 || - check(_PyTok_CursorSetOffset(&cursor, source.len) == 0, - "cannot set cursor to EOF") < 0 || - check(cursor.lineno == 4 && cursor.pos == source.len, - "wrong cursor at EOF") < 0) { - goto error; - } - -#if SIZEOF_VOID_P > 4 - char byte = 0; - _PyTok_SourceText huge_source = { - .bytes = &byte, - .len = (_PyTok_Off)INT_MAX + 1, - }; - _PyTok_Cursor huge_cursor = { - .source = &huge_source, - .pos = INT_MAX, - .line_end = (_PyTok_Off)INT_MAX + 1, - .lineno = 1, - }; - if (check(_PyTok_CursorAdvance(&huge_cursor) == EOF && - huge_cursor.pos == INT_MAX, - "cursor advanced past maximum column") < 0) { - goto error; - } -#endif - - _PyTok_Off base = source.len; - _PyTok_SourceDiscard(&source); - if (_PyTok_SourceAppendLine(&source, "ab\n", 3, 0) < 0 || - _PyTok_SourceAppendLine(&source, "cd", 2, 0) < 0) { + "appended after unterminated source line") < 0) { goto error; } - _PyTok_CursorInit(&cursor, &source); - if (_PyTok_CursorSetLine(&cursor, 1) < 0 || - check(cursor.pos == base && _PyTok_CursorPeek(&cursor, 1) == 'b', - "wrong retained cursor line") < 0 || - _PyTok_CursorSetLine(&cursor, 2) < 0 || - check(_PyTok_CursorAdvance(&cursor) == 'c', - "wrong retained cursor byte") < 0 || - _PyTok_CursorSetOffset(&cursor, base + 5) < 0 || - check(cursor.lineno == 2 && _PyTok_CursorAdvance(&cursor) == EOF, - "wrong retained cursor EOF") < 0) { + if (check_line_view(&source, 1, "tail") < 0 || + check_line_view(&source, PY_SSIZE_T_MAX, "tail") < 0) { goto error; } @@ -410,7 +165,6 @@ test_tokenizer_source_discard(PyObject *Py_UNUSED(module), static PyMethodDef test_methods[] = { {"test_tokenizer_source", test_tokenizer_source, METH_NOARGS}, - {"test_tokenizer_cursor", test_tokenizer_cursor, METH_NOARGS}, {"test_tokenizer_source_discard", test_tokenizer_source_discard, METH_NOARGS}, {NULL}, }; diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 70c54e0e41efc63..8c9fd0b0acc6f05 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -183,6 +183,7 @@ + diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index b0799b8dc9ecddb..39d5db39fde3897 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -463,6 +463,9 @@ Source Files + + Source Files + Source Files diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index cd37db739301923..33310717b96b714 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -421,9 +421,7 @@ - - @@ -591,9 +589,9 @@ + - diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index e4b9039eec13fc0..c493b4ad8aae51b 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -321,18 +321,12 @@ Objects - - Parser - Parser Parser - - Parser - Parser @@ -1352,6 +1346,9 @@ Parser + + Parser + Parser @@ -1361,9 +1358,6 @@ Parser - - Parser - Parser diff --git a/Parser/lexer/layout.c b/Parser/lexer/layout.c new file mode 100644 index 000000000000000..6091f9adeac958c --- /dev/null +++ b/Parser/lexer/layout.c @@ -0,0 +1,195 @@ +#include "Python.h" +#include "errcode.h" +#include "pycore_token.h" + +#include "lexer_internal.h" +#include "../tokenizer/helpers.h" +#include "../tokenizer/reader.h" + +#define TABSIZE 8 +#define ALTTABSIZE 1 + +int +_PyLexer_ContinueLine(struct tok_state *tok) +{ + int c = tok_nextc(tok); + if (c == '\r') { + c = tok_nextc(tok); + } + if (c != '\n') { + tok->done = E_LINECONT; + return -1; + } + c = tok_nextc(tok); + if (c == EOF) { + tok->done = E_EOF; + tok->cur = tok->inp; + return -1; + } else { + tok_backup(tok, c); + } + return c; +} + + +static int +update_indentation(struct tok_state *tok, int col, int altcol) +{ + lexer_layout_state *layout = &tok->layout; + if (col == layout->stack[layout->depth].column) { + if (altcol != layout->stack[layout->depth].alternate_column) { + _PyTokenizer_indenterror(tok); + return -1; + } + } + else if (col > layout->stack[layout->depth].column) { + if (layout->depth + 1 >= MAXINDENT) { + tok->done = E_TOODEEP; + tok->cur = tok->inp; + return -1; + } + if (altcol <= layout->stack[layout->depth].alternate_column) { + _PyTokenizer_indenterror(tok); + return -1; + } + layout->pending++; + layout->stack[++layout->depth] = (indentation_level){col, altcol}; + } + else { + while (layout->depth > 0 && + col < layout->stack[layout->depth].column) { + layout->pending--; + layout->depth--; + } + if (col != layout->stack[layout->depth].column) { + tok->done = E_DEDENT; + tok->cur = tok->inp; + return -1; + } + if (altcol != layout->stack[layout->depth].alternate_column) { + _PyTokenizer_indenterror(tok); + return -1; + } + } + return 0; +} + +int +_PyLexer_BeginLine(struct tok_state *tok) +{ + assert(tok->layout.at_bol); + int c; + int blankline = 0; + int col = 0; + int altcol = 0; + tok->layout.at_bol = 0; + int cont_line_col = 0; + for (;;) { + c = tok_nextc(tok); + if (c == ' ') { + col++, altcol++; + } + else if (c == '\t') { + col = (col / TABSIZE + 1) * TABSIZE; + altcol = (altcol / ALTTABSIZE + 1) * ALTTABSIZE; + } + else if (c == '\014') {/* Control-L (formfeed) */ + col = altcol = 0; /* For Emacs users */ + } + else if (c == '\\') { + // Indentation cannot be split over multiple physical lines + // using backslashes. This means that if we found a backslash + // preceded by whitespace, **the first one we find** determines + // the level of indentation of whatever comes next. + cont_line_col = cont_line_col ? cont_line_col : col; + if ((c = _PyLexer_ContinueLine(tok)) == -1) { + return -1; + } + } + else if (c == EOF && PyErr_Occurred()) { + return -1; + } + else { + break; + } + } + tok_backup(tok, c); + if (c == '#' || c == '\n' || c == '\r') { + int interactive = _PyTok_ReaderIsInteractive(tok); + /* Lines with only whitespace and/or comments + shouldn't affect the indentation and are + not passed to the parser as NEWLINE tokens, + except *totally* empty lines in interactive + mode, which signal the end of a command group. */ + if (col == 0 && c == '\n' && interactive) { + blankline = 0; /* Let it through */ + } + else if (interactive && tok->lineno == 1) { + /* In interactive mode, if the first line contains + only spaces and/or a comment, let it through. */ + blankline = 0; + col = altcol = 0; + } + else { + blankline = 1; /* Ignore completely */ + } + } + if (!blankline && tok->level == 0) { + col = cont_line_col ? cont_line_col : col; + altcol = cont_line_col ? cont_line_col : altcol; + if (update_indentation(tok, col, altcol) < 0) { + return -1; + } + } + return blankline; +} + +int +_PyLexer_IndentationToken(struct tok_state *tok, struct token *token) +{ + assert(tok->layout.pending != 0); + _PyTok_Off p_start = -1; + _PyTok_Off p_end = -1; + if (tok->layout.pending < 0) { + if (tok->tok_extra_tokens) { + p_start = tok->cur; + p_end = tok->cur; + } + tok->layout.pending++; + return _PyLexer_token_setup(tok, token, DEDENT, p_start, p_end); + } + else { + if (tok->tok_extra_tokens) { + p_start = tok->buf_offset; + p_end = tok->cur; + } + tok->layout.pending--; + return _PyLexer_token_setup(tok, token, INDENT, p_start, p_end); + } +} + +int +_PyLexer_Newline(struct tok_state *tok, struct token *token, int blankline) +{ + tok->layout.at_bol = 1; + if (blankline || tok->level > 0) { + if (!tok->tok_extra_tokens) { + return 0; + } + } + else if (!tok->layout.comment_newline || !tok->tok_extra_tokens) { + return _PyLexer_token_setup(tok, token, NEWLINE, + tok->start, tok->cur - 1); + } + tok->layout.comment_newline = 0; + return _PyLexer_token_setup(tok, token, NL, tok->start, tok->cur); +} + +void +_PyLexer_ImplyDedents(struct tok_state *tok) +{ + if (tok->layout.depth != 0) { + tok->layout.pending = -tok->layout.depth; + tok->layout.depth = 0; + } +} diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index 110a225750f0550..f27fee8d61b9085 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -7,10 +7,6 @@ #include "../tokenizer/helpers.h" #include "../tokenizer/reader.h" -#define TABSIZE 8 -#define ALTTABSIZE 1 - - #define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end) /* Spaces in this constant are treated as "zero or more spaces or tabs" when @@ -93,6 +89,7 @@ verify_identifier(struct tok_state *tok) assert(PyUnicode_GET_LENGTH(s) > 0); if (invalid < PyUnicode_GET_LENGTH(s)) { Py_UCS4 ch = PyUnicode_READ_CHAR(s, invalid); + _PyTok_Off error_cursor = tok->cur; if (invalid + 1 < PyUnicode_GET_LENGTH(s)) { /* Determine the offset in UTF-8 encoded input */ Py_SETREF(s, PyUnicode_Substring(s, 0, invalid + 1)); @@ -103,14 +100,20 @@ verify_identifier(struct tok_state *tok) tok->done = E_ERROR; return 0; } - tok->cur = tok->start + PyBytes_GET_SIZE(s); + error_cursor = tok->start + PyBytes_GET_SIZE(s); } Py_DECREF(s); if (Py_UNICODE_ISPRINTABLE(ch)) { - _PyTokenizer_syntaxerror(tok, "invalid character '%c' (U+%04X)", ch, ch); + _PyTokenizer_syntaxerror_at( + tok, _PyLexer_BufferPointer(tok, tok->line_start), + error_cursor - tok->line_start, tok->lineno, -1, -1, + "invalid character '%c' (U+%04X)", ch, ch); } else { - _PyTokenizer_syntaxerror(tok, "invalid non-printable character U+%04X", ch); + _PyTokenizer_syntaxerror_at( + tok, _PyLexer_BufferPointer(tok, tok->line_start), + error_cursor - tok->line_start, tok->lineno, -1, -1, + "invalid non-printable character U+%04X", ch); } return 0; } @@ -118,31 +121,6 @@ verify_identifier(struct tok_state *tok) return 1; } - - -static inline int -tok_continuation_line(struct tok_state *tok) { - int c = tok_nextc(tok); - if (c == '\r') { - c = tok_nextc(tok); - } - if (c != '\n') { - tok->done = E_LINECONT; - return -1; - } - c = tok_nextc(tok); - if (c == EOF) { - tok->done = E_EOF; - tok->cur = tok->inp; - return -1; - } else { - tok_backup(tok, c); - } - return c; -} - - - int _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token *token) { @@ -160,102 +138,10 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token blankline = 0; - /* Get indentation level */ - if (tok->atbol) { - int col = 0; - int altcol = 0; - tok->atbol = 0; - int cont_line_col = 0; - for (;;) { - c = tok_nextc(tok); - if (c == ' ') { - col++, altcol++; - } - else if (c == '\t') { - col = (col / TABSIZE + 1) * TABSIZE; - altcol = (altcol / ALTTABSIZE + 1) * ALTTABSIZE; - } - else if (c == '\014') {/* Control-L (formfeed) */ - col = altcol = 0; /* For Emacs users */ - } - else if (c == '\\') { - // Indentation cannot be split over multiple physical lines - // using backslashes. This means that if we found a backslash - // preceded by whitespace, **the first one we find** determines - // the level of indentation of whatever comes next. - cont_line_col = cont_line_col ? cont_line_col : col; - if ((c = tok_continuation_line(tok)) == -1) { - return MAKE_TOKEN(ERRORTOKEN); - } - } - else if (c == EOF && PyErr_Occurred()) { - return MAKE_TOKEN(ERRORTOKEN); - } - else { - break; - } - } - tok_backup(tok, c); - if (c == '#' || c == '\n' || c == '\r') { - /* Lines with only whitespace and/or comments - shouldn't affect the indentation and are - not passed to the parser as NEWLINE tokens, - except *totally* empty lines in interactive - mode, which signal the end of a command group. */ - if (col == 0 && c == '\n' && tok->prompt != NULL) { - blankline = 0; /* Let it through */ - } - else if (tok->prompt != NULL && tok->lineno == 1) { - /* In interactive mode, if the first line contains - only spaces and/or a comment, let it through. */ - blankline = 0; - col = altcol = 0; - } - else { - blankline = 1; /* Ignore completely */ - } - /* We can't jump back right here since we still - may need to skip to the end of a comment */ - } - if (!blankline && tok->level == 0) { - col = cont_line_col ? cont_line_col : col; - altcol = cont_line_col ? cont_line_col : altcol; - if (col == tok->indstack[tok->indent]) { - /* No change */ - if (altcol != tok->altindstack[tok->indent]) { - return MAKE_TOKEN(_PyTokenizer_indenterror(tok)); - } - } - else if (col > tok->indstack[tok->indent]) { - /* Indent -- always one */ - if (tok->indent+1 >= MAXINDENT) { - tok->done = E_TOODEEP; - tok->cur = tok->inp; - return MAKE_TOKEN(ERRORTOKEN); - } - if (altcol <= tok->altindstack[tok->indent]) { - return MAKE_TOKEN(_PyTokenizer_indenterror(tok)); - } - tok->pendin++; - tok->indstack[++tok->indent] = col; - tok->altindstack[tok->indent] = altcol; - } - else /* col < tok->indstack[tok->indent] */ { - /* Dedent -- any number, must be consistent */ - while (tok->indent > 0 && - col < tok->indstack[tok->indent]) { - tok->pendin--; - tok->indent--; - } - if (col != tok->indstack[tok->indent]) { - tok->done = E_DEDENT; - tok->cur = tok->inp; - return MAKE_TOKEN(ERRORTOKEN); - } - if (altcol != tok->altindstack[tok->indent]) { - return MAKE_TOKEN(_PyTokenizer_indenterror(tok)); - } - } + if (tok->layout.at_bol) { + blankline = _PyLexer_BeginLine(tok); + if (blankline < 0) { + return MAKE_TOKEN(ERRORTOKEN); } } @@ -263,24 +149,8 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token tok->start_loc = (_PyTok_Loc){ tok->lineno, tok->line_start >= 0 ? _PyLexer_ByteColumn(tok) : -1}; - /* Return pending indents/dedents */ - if (tok->pendin != 0) { - if (tok->pendin < 0) { - if (tok->tok_extra_tokens) { - p_start = tok->cur; - p_end = tok->cur; - } - tok->pendin++; - return MAKE_TOKEN(DEDENT); - } - else { - if (tok->tok_extra_tokens) { - p_start = tok->buf_offset; - p_end = tok->cur; - } - tok->pendin--; - return MAKE_TOKEN(INDENT); - } + if (tok->layout.pending != 0) { + return _PyLexer_IndentationToken(tok, token); } /* Peek ahead at the next character */ @@ -373,7 +243,7 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token /* If this type ignore is the only thing on the line, consume the newline also. */ if (blankline) { tok_nextc(tok); - tok->atbol = 1; + tok->layout.at_bol = 1; } } else { p_start = _PyLexer_BufferOffset(tok, type_start); @@ -389,7 +259,7 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token tok_backup(tok, c); /* don't eat the newline or EOF */ p_start = _PyLexer_BufferOffset(tok, p); p_end = tok->cur; - tok->comment_newline = blankline; + tok->layout.comment_newline = blankline; return MAKE_TOKEN(COMMENT); } } @@ -470,29 +340,12 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token c = tok_nextc(tok); } - /* Newline */ if (c == '\n') { - tok->atbol = 1; - if (blankline || tok->level > 0) { - if (tok->tok_extra_tokens) { - if (tok->comment_newline) { - tok->comment_newline = 0; - } - p_start = tok->start; - p_end = tok->cur; - return MAKE_TOKEN(NL); - } + int type = _PyLexer_Newline(tok, token, blankline); + if (type == 0) { goto nextline; } - if (tok->comment_newline && tok->tok_extra_tokens) { - tok->comment_newline = 0; - p_start = tok->start; - p_end = tok->cur; - return MAKE_TOKEN(NL); - } - p_start = tok->start; - p_end = tok->cur - 1; /* Leave '\n' out of the string */ - return MAKE_TOKEN(NEWLINE); + return type; } /* Period or number starting with period? */ @@ -533,7 +386,7 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token /* Line continuation */ if (c == '\\') { - if ((c = tok_continuation_line(tok)) == -1) { + if ((c = _PyLexer_ContinueLine(tok)) == -1) { return MAKE_TOKEN(ERRORTOKEN); } goto again; /* Read next line */ diff --git a/Parser/lexer/lexer.h b/Parser/lexer/lexer.h deleted file mode 100644 index 7302198cc91010b..000000000000000 --- a/Parser/lexer/lexer.h +++ /dev/null @@ -1,6 +0,0 @@ -#ifndef _PY_LEXER_LEXER_H_ -#define _PY_LEXER_LEXER_H_ - -#include "state.h" - -#endif diff --git a/Parser/lexer/lexer_internal.h b/Parser/lexer/lexer_internal.h index 210d182d42dd9f5..f0d9576d95f8ff3 100644 --- a/Parser/lexer/lexer_internal.h +++ b/Parser/lexer/lexer_internal.h @@ -2,7 +2,7 @@ #define _PY_LEXER_INTERNAL_H_ #include "errcode.h" -#include "lexer.h" +#include "state.h" #define is_potential_identifier_start(c) (\ (c >= 'a' && c <= 'z')\ @@ -50,6 +50,12 @@ tok_nextc(struct tok_state *tok) tok->source.bytes[tok->cur++ - tok->source.base_offset]); } +/* Return -1 on error, otherwise whether the line is blank. */ +int _PyLexer_BeginLine(struct tok_state *); +int _PyLexer_ContinueLine(struct tok_state *); +int _PyLexer_IndentationToken(struct tok_state *, struct token *); +/* Return zero when the newline is suppressed, otherwise its token type. */ +int _PyLexer_Newline(struct tok_state *, struct token *, int); void _PyLexer_backup(struct tok_state *, int); int _PyLexer_record_ftstring_comment( struct tok_state *, ftstring_state *, _PyTok_Off, _PyTok_Off); diff --git a/Parser/lexer/state.c b/Parser/lexer/state.c index a6617c33480c855..75ff26f16d47ba7 100644 --- a/Parser/lexer/state.c +++ b/Parser/lexer/state.c @@ -1,5 +1,4 @@ #include "Python.h" -#include "pycore_pystate.h" #include "pycore_token.h" #include "errcode.h" @@ -7,55 +6,6 @@ #include "../tokenizer/helpers.h" #include "../tokenizer/reader.h" -/* Create and initialize a new tok_state structure */ -struct tok_state * -_PyTokenizer_tok_new(void) -{ - struct tok_state *tok = (struct tok_state *)PyMem_Calloc( - 1, - sizeof(struct tok_state)); - if (tok == NULL) { - PyErr_NoMemory(); - return NULL; - } - - tok->cur = tok->inp = 0; - tok->line_start = -1; - tok->fp_interactive = 0; - tok->interactive_src_start = NULL; - tok->interactive_src_end = NULL; - tok->start = -1; - tok->done = E_OK; - tok->fp = NULL; - tok->indent = 0; - tok->indstack[0] = 0; - tok->atbol = 1; - tok->pendin = 0; - tok->prompt = NULL; - tok->lineno = 0; - tok->start_loc = (_PyTok_Loc){-1, -1}; - tok->level = 0; - tok->altindstack[0] = 0; - tok->encoding = NULL; - tok->filename = NULL; - tok->module = NULL; - tok->type_comments = 0; - tok->interactive_underflow = IUNDERFLOW_NORMAL; - tok->str = NULL; - tok->report_warnings = 1; - tok->tok_extra_tokens = 0; - tok->comment_newline = 0; - tok->implicit_newline = 0; - _PyTok_SourceInit(&tok->source); - tok->reader = NULL; - tok->ftstring_stack = tok->ftstring_stack_inline; - tok->ftstring_capacity = FTSTRING_STACK_INLINE_CAPACITY; -#ifdef Py_DEBUG - tok->debug = _Py_GetConfig()->parser_debug; -#endif - return tok; -} - ftstring_state * _PyLexer_PushFTString(struct tok_state *tok) { diff --git a/Parser/lexer/state.h b/Parser/lexer/state.h index 0f9ddb6d45e9611..dff67c2ba83dae4 100644 --- a/Parser/lexer/state.h +++ b/Parser/lexer/state.h @@ -10,15 +10,6 @@ #define MAXFTSTRINGLEVEL 150 #define FTSTRING_STACK_INLINE_CAPACITY 1 -enum interactive_underflow_t { - /* Normal mode of operation: return a new token when asked in interactive mode */ - IUNDERFLOW_NORMAL, - /* Forcefully return ENDMARKER when asked for a new token in interactive mode. This - * can be used to prevent the tokenizer to prompt the user for new tokens */ - IUNDERFLOW_STOP, -}; - - typedef enum { FTSTRING_MODE_MIDDLE, FTSTRING_MODE_EXPRESSION, @@ -66,6 +57,19 @@ _PyLexer_IsRawString(ftstring_kind kind) return kind == RAW_FSTRING || kind == RAW_TSTRING; } +typedef struct { + int column; + int alternate_column; +} indentation_level; + +typedef struct { + int depth; + int pending; + int at_bol; + int comment_newline; + indentation_level stack[MAXINDENT]; +} lexer_layout_state; + /* Tokenizer state */ struct tok_state { _PyTok_Off buf_offset; @@ -74,19 +78,13 @@ struct tok_state { _PyTok_Off start; _PyTok_Off line_start; _PyTok_SourceText source; - int fp_interactive; /* If the file descriptor is interactive */ - char *interactive_src_start; /* The start of the source parsed so far in interactive mode */ - char *interactive_src_end; /* The end of the source parsed so far in interactive mode */ int done; /* E_OK normally, E_EOF at EOF, otherwise error code */ /* NB If done != E_OK, cur must be == inp!!! */ FILE *fp; /* Rest of input; NULL if tokenizing a string */ - int indent; /* Current indentation index */ - int indstack[MAXINDENT]; /* Stack of indents */ - int atbol; /* Nonzero if at begin of new line */ - int pendin; /* Pending indents (if > 0) or dedents (if < 0) */ - const char *prompt; /* For interactive prompting */ + lexer_layout_state layout; int lineno; /* Current line number */ _PyTok_Loc start_loc; + _PyTokenizer_Diagnostic diagnostic; int level; /* () [] {} Parentheses nesting level */ /* Used to allow free continuations inside them */ char parenstack[MAXLEVEL]; @@ -94,25 +92,18 @@ struct tok_state { int parencolstack[MAXLEVEL]; PyObject *filename; PyObject *module; - /* Stuff for checking on different tab sizes */ - int altindstack[MAXINDENT]; /* Stack of alternate indents */ /* Stuff for PEP 0263 */ char *encoding; /* Source encoding. */ - char* str; /* Source string being tokenized (if tokenizing from a string)*/ struct _PyTok_Reader *reader; int type_comments; /* Whether to look for type comments */ - /* How to proceed when asked for a new token in interactive mode */ - enum interactive_underflow_t interactive_underflow; - int report_warnings; ftstring_state *ftstring_stack; ftstring_state ftstring_stack_inline[FTSTRING_STACK_INLINE_CAPACITY]; int ftstring_depth; int ftstring_capacity; int tok_extra_tokens; - int comment_newline; int implicit_newline; #ifdef Py_DEBUG int debug; @@ -182,7 +173,8 @@ _PyLexer_ByteColumn(const struct tok_state *tok) int _PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, _PyTok_Off start, _PyTok_Off end); -struct tok_state *_PyTokenizer_tok_new(void); +void _PyLexer_ImplyDedents(struct tok_state *); + void _PyTokenizer_Free(struct tok_state *); ftstring_state *_PyLexer_PushFTString(struct tok_state *); void _PyLexer_PopFTString(struct tok_state *); diff --git a/Parser/lexer/string.c b/Parser/lexer/string.c index 945480ef86f7789..2aef82997f4ffd7 100644 --- a/Parser/lexer/string.c +++ b/Parser/lexer/string.c @@ -7,13 +7,18 @@ #define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end) -static void -rewind_to_string_start(struct tok_state *tok, _PyTok_Off start, - _PyTok_Loc location) +static int +string_error_token(struct tok_state *tok, struct token *token, + _PyTok_Off start, _PyTok_Loc location) { - tok->cur = start + 1; - tok->line_start = start - location.byte_col; - tok->lineno = location.lineno; + tok->diagnostic = (_PyTokenizer_Diagnostic){ + .location = {location.lineno, location.byte_col + 1}, + .text_span = _PyTok_SpanFromBounds(start - location.byte_col, tok->inp), + }; + int type = _PyLexer_token_setup(tok, token, ERRORTOKEN, -1, -1); + token->start_loc = location; + token->end_loc = (_PyTok_Loc){location.lineno, -1}; + return type; } int @@ -351,44 +356,51 @@ _PyLexer_scan_string(struct tok_state *tok, struct token *token, int c) } if (c == EOF || (quote_size == 1 && c == '\n')) { int end_lineno = tok->lineno; - rewind_to_string_start(tok, tok->start, tok->start_loc); + _PyTok_Loc location = tok->start_loc; + const char *line = _PyLexer_BufferPointer(tok, tok->start) - location.byte_col; + Py_ssize_t cursor_offset = (Py_ssize_t)location.byte_col + 1; const ftstring_state *state = _PyLexer_CurrentFTString(tok); if (state != NULL) { /* A matching quote belongs to the surrounding formatted * string, so the expression is missing its closing brace. */ if (state->quote == quote && state->quote_size == quote_size) { - return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, + _PyTokenizer_syntaxerror_at( + tok, line, cursor_offset, location.lineno, -1, -1, "%c-string: expecting '}'", - _PyLexer_StringPrefix(state->kind))); + _PyLexer_StringPrefix(state->kind)); + return string_error_token(tok, token, tok->start, location); } } if (quote_size == 3) { - _PyTokenizer_syntaxerror(tok, "unterminated triple-quoted string literal" - " (detected at line %d)", end_lineno); + _PyTokenizer_syntaxerror_at( + tok, line, cursor_offset, location.lineno, -1, -1, + "unterminated triple-quoted string literal" + " (detected at line %d)", end_lineno); if (c != '\n') { tok->done = E_EOFS; } - return MAKE_TOKEN(ERRORTOKEN); + return string_error_token(tok, token, tok->start, location); } else { if (has_escaped_quote) { - _PyTokenizer_syntaxerror( - tok, + _PyTokenizer_syntaxerror_at( + tok, line, cursor_offset, location.lineno, -1, -1, "unterminated string literal (detected at line %d); " "perhaps you escaped the end quote?", end_lineno ); } else { - _PyTokenizer_syntaxerror( - tok, "unterminated string literal (detected at line %d)", end_lineno + _PyTokenizer_syntaxerror_at( + tok, line, cursor_offset, location.lineno, -1, -1, + "unterminated string literal (detected at line %d)", end_lineno ); } if (c != '\n') { tok->done = E_EOLS; } - return MAKE_TOKEN(ERRORTOKEN); + return string_error_token(tok, token, tok->start, location); } } if (c == quote) { @@ -452,25 +464,29 @@ _PyLexer_get_ftstring(struct tok_state *tok, ftstring_state *current, struct tok } int end_lineno = tok->lineno; - rewind_to_string_start(tok, - current->start, - current->start_loc); + _PyTok_Loc location = current->start_loc; + const char *line = _PyLexer_BufferPointer(tok, current->start) - location.byte_col; + Py_ssize_t cursor_offset = (Py_ssize_t)location.byte_col + 1; if (quote_size == 3) { - _PyTokenizer_syntaxerror(tok, - "unterminated triple-quoted %c-string literal" - " (detected at line %d)", - _PyLexer_StringPrefix(current->kind), end_lineno); + _PyTokenizer_syntaxerror_at( + tok, line, cursor_offset, location.lineno, -1, -1, + "unterminated triple-quoted %c-string literal" + " (detected at line %d)", + _PyLexer_StringPrefix(current->kind), end_lineno); if (c != '\n') { tok->done = E_EOFS; } - return MAKE_TOKEN(ERRORTOKEN); + return string_error_token(tok, token, + current->start, location); } else { - return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, - "unterminated %c-string literal (detected at" - " line %d)", - _PyLexer_StringPrefix(current->kind), end_lineno)); + _PyTokenizer_syntaxerror_at( + tok, line, cursor_offset, location.lineno, -1, -1, + "unterminated %c-string literal (detected at line %d)", + _PyLexer_StringPrefix(current->kind), end_lineno); + return string_error_token(tok, token, + current->start, location); } } diff --git a/Parser/pegen_errors.c b/Parser/pegen_errors.c index 7841f01b612915c..74ab56c912ae1af 100644 --- a/Parser/pegen_errors.c +++ b/Parser/pegen_errors.c @@ -206,7 +206,9 @@ _PyPegen_raise_error(Parser *p, PyObject *errtype, int use_mark, const char *err Py_ssize_t end_col_offset = -1; if (t->col_offset == -1) { _PyTokenizer_Info info = _PyTokenizer_GetInfo(p->tok); - if (info.cursor == info.input_span.start) { + if (info.diagnostic.location.lineno != 0) { + col_offset = info.diagnostic.location.byte_col; + } else if (info.cursor == info.input_span.start) { col_offset = 0; } else { col_offset = Py_SAFE_DOWNCAST( @@ -256,8 +258,10 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype, PyObject *tmp = NULL; p->error_indicator = 1; _PyTokenizer_Info info = _PyTokenizer_GetInfo(p->tok); - _PyTok_Loc location = info.location; - _PyTok_Span text_span = info.line_span; + _PyTok_Loc location = info.diagnostic.location.lineno != 0 + ? info.diagnostic.location : info.location; + _PyTok_Span text_span = info.diagnostic.location.lineno != 0 + ? info.diagnostic.text_span : info.line_span; if (end_lineno == CURRENT_POS) { end_lineno = location.lineno; diff --git a/Parser/tokenizer/api.c b/Parser/tokenizer/api.c index 67e129d52441c76..3f5efa8dc8f26e6 100644 --- a/Parser/tokenizer/api.c +++ b/Parser/tokenizer/api.c @@ -4,7 +4,6 @@ #include "tokenizer.h" #include "reader.h" -#include "reader_internal.h" #include "../lexer/state.h" _PyTokenizer_Info @@ -12,6 +11,7 @@ _PyTokenizer_GetInfo(const struct tok_state *tok) { _PyTokenizer_Info info = { .status = tok->done, + .diagnostic = tok->diagnostic, .location = {tok->lineno, tok->line_start < 0 ? -1 : (int)(tok->cur - tok->line_start)}, .cursor = tok->cur, @@ -20,7 +20,7 @@ _PyTokenizer_GetInfo(const struct tok_state *tok) .level = tok->level, .delimiter_loc = {-1, -1}, .in_formatted_string = tok->ftstring_depth != 0, - .is_interactive = tok->reader->kind == _PYTOK_READER_INTERACTIVE, + .is_interactive = _PyTok_ReaderIsInteractive(tok), .is_file = tok->fp != NULL && tok->fp != stdin, .filename = tok->filename, .module = tok->module, @@ -81,32 +81,7 @@ const char * _PyTokenizer_LineView(const struct tok_state *tok, Py_ssize_t lineno, Py_ssize_t *length) { - const char *line = _PyTokenizer_RetainedSource(tok); - if (line == NULL) { - line = _PyLexer_BufferPointer(tok, tok->buf_offset); - } - for (Py_ssize_t i = 1; i < lineno; i++) { - const char *next = strchr(line, '\n'); - if (next == NULL) { - break; - } - line = next + 1; - } - const char *end = strchr(line, '\n'); - *length = end != NULL ? end - line : (Py_ssize_t)strlen(line); - return line; -} - -const char * -_PyTokenizer_RetainedSource(const struct tok_state *tok) -{ - if (tok->reader->kind == _PYTOK_READER_PREPARED) { - return _PyTok_SourceData(&tok->source); - } - if (tok->reader->kind == _PYTOK_READER_INTERACTIVE) { - return tok->source.bytes; - } - return NULL; + return _PyTok_SourceLineView(&tok->source, lineno, length); } void @@ -130,10 +105,7 @@ _PyTokenizer_SetOptions(struct tok_state *tok, int extra_tokens, void _PyTokenizer_ImplyDedents(struct tok_state *tok) { - if (tok->indent != 0) { - tok->pendin = -tok->indent; - tok->indent = 0; - } + _PyLexer_ImplyDedents(tok); } int @@ -160,11 +132,11 @@ _PyTokenizer_HasTrailingStatement(const struct tok_state *tok) int _PyTokenizer_IsInteractive(const struct tok_state *tok) { - return tok->prompt != NULL; + return _PyTok_ReaderIsInteractive(tok); } void _PyTokenizer_StopInteractive(struct tok_state *tok) { - tok->interactive_underflow = IUNDERFLOW_STOP; + _PyTok_ReaderStopInteractive(tok); } diff --git a/Parser/tokenizer/cursor.c b/Parser/tokenizer/cursor.c deleted file mode 100644 index 523b99dedc6160a..000000000000000 --- a/Parser/tokenizer/cursor.c +++ /dev/null @@ -1,82 +0,0 @@ -#include "Python.h" - -#include "cursor.h" - -static void -set_line(_PyTok_Cursor *cursor, int lineno, _PyTok_Off start, - _PyTok_Off end) -{ - cursor->pos = start; - cursor->line_start = start; - cursor->line_end = end; - cursor->lineno = lineno; -} - -int -_PyTok_CursorSetLine(_PyTok_Cursor *cursor, int lineno) -{ - if (cursor->source == NULL) { - PyErr_SetString(PyExc_SystemError, "cursor has no tokenizer source"); - return -1; - } - const _PyTok_SourceText *source = cursor->source; - if (lineno > 0 && cursor->lineno == lineno - 1 && - lineno <= source->nlines) { - _PyTok_Off start = cursor->line_end; - _PyTok_Off end = source->base_offset + source->len; - if (lineno < source->nlines) { - end = _PyTok_SourceFindLineEnd(source, start); - if (end < 0) { - return -1; - } - } - set_line(cursor, lineno, start, end); - return 0; - } - - _PyTok_Line line; - if (_PyTok_SourceLine(source, lineno, &line) < 0) { - return -1; - } - set_line(cursor, lineno, line.start, line.end); - return 0; -} - -int -_PyTok_CursorSetOffset(_PyTok_Cursor *cursor, _PyTok_Off offset) -{ - if (cursor->source == NULL) { - PyErr_SetString(PyExc_SystemError, "cursor has no tokenizer source"); - return -1; - } - const _PyTok_SourceText *source = cursor->source; - int stays_on_line = cursor->lineno > 0 && - offset >= cursor->line_start && offset < cursor->line_end; - if (!stays_on_line && cursor->lineno > 0 && - offset == cursor->line_end && - offset - source->base_offset == source->len && - (source->len == 0 || source->bytes[source->len - 1] != '\n')) { - stays_on_line = 1; - } - if (stays_on_line) { - cursor->pos = offset; - return 0; - } - - _PyTok_Loc loc; - if (_PyTok_SourceLocation( - source, offset, _PYTOK_AFFINITY_RIGHT, &loc) < 0) { - return -1; - } - _PyTok_Off start = offset - loc.byte_col; - _PyTok_Off end = source->base_offset + source->len; - if (loc.lineno < source->nlines) { - end = _PyTok_SourceFindLineEnd(source, start); - if (end < 0) { - return -1; - } - } - set_line(cursor, loc.lineno, start, end); - cursor->pos = offset; - return 0; -} diff --git a/Parser/tokenizer/cursor.h b/Parser/tokenizer/cursor.h deleted file mode 100644 index 18e404251316f0c..000000000000000 --- a/Parser/tokenizer/cursor.h +++ /dev/null @@ -1,73 +0,0 @@ -#ifndef Py_TOKENIZER_CURSOR_H -#define Py_TOKENIZER_CURSOR_H - -#include "source.h" - -typedef struct { - /* The source must remain initialized at this address while in use. */ - const _PyTok_SourceText *source; - _PyTok_Off pos; - _PyTok_Off line_start; - _PyTok_Off line_end; - int lineno; -} _PyTok_Cursor; - -/* Move to the start of a 1-based line. Both setters preserve the cursor on - error. */ -PyAPI_FUNC(int) _PyTok_CursorSetLine(_PyTok_Cursor *, int); -/* Move to an offset. A line boundary selects the following line. */ -PyAPI_FUNC(int) _PyTok_CursorSetOffset(_PyTok_Cursor *, _PyTok_Off); - -static inline void -_PyTok_CursorInit(_PyTok_Cursor *cursor, const _PyTok_SourceText *source) -{ - _PyTok_Off base = source != NULL ? source->base_offset : 0; - *cursor = (_PyTok_Cursor){ - .source = source, - .pos = base, - .line_start = base, - .line_end = base, - }; -} - -/* Read one byte from the current line, including its terminating newline. - EOF marks the line boundary, not necessarily the end of the source. It is - also returned if advancing would make the byte column unrepresentable. */ -static inline int -_PyTok_CursorAdvance(_PyTok_Cursor *cursor) -{ - assert(cursor->source != NULL); - assert(cursor->pos >= cursor->line_start); - assert(cursor->pos <= cursor->line_end); - assert(cursor->line_start >= cursor->source->base_offset); - assert(cursor->line_end - cursor->source->base_offset <= cursor->source->len); - if (cursor->pos >= cursor->line_end) { - return EOF; - } - if (cursor->pos - cursor->line_start >= INT_MAX) { - return EOF; - } - return Py_CHARMASK(cursor->source->bytes[ - cursor->pos++ - cursor->source->base_offset]); -} - -/* Return the byte at a nonnegative distance within the current line, or EOF - if the distance reaches or crosses the line boundary. */ -static inline int -_PyTok_CursorPeek(const _PyTok_Cursor *cursor, int distance) -{ - assert(cursor->source != NULL); - assert(cursor->pos >= cursor->line_start); - assert(cursor->pos <= cursor->line_end); - assert(cursor->line_start >= cursor->source->base_offset); - assert(cursor->line_end - cursor->source->base_offset <= cursor->source->len); - assert(distance >= 0); - if (distance < 0 || - distance >= cursor->line_end - cursor->pos) { - return EOF; - } - return Py_CHARMASK(cursor->source->bytes[ - cursor->pos - cursor->source->base_offset + distance]); -} - -#endif diff --git a/Parser/tokenizer/decoder.c b/Parser/tokenizer/decoder.c index 5b588572f1fb7e8..c36860237fce0b9 100644 --- a/Parser/tokenizer/decoder.c +++ b/Parser/tokenizer/decoder.c @@ -103,7 +103,9 @@ _PyTok_NormalizeNewlines(const char *data, Py_ssize_t len, int preserve_crlf, } result[write] = '\0'; *out_len = write; - *implicit_newline = implicit; + if (implicit_newline != NULL) { + *implicit_newline = implicit; + } return result; } @@ -244,7 +246,8 @@ _PyTok_DetectEncoding(struct tok_state *tok, const _PyTok_Chunk *first, end_col--; } _PyTokenizer_syntaxerror_at( - tok, line_data, 0, cookie_line, 0, end_col, "encoding problem: %s with BOM", cookie); + tok, line_data, 0, cookie_line, 0, end_col, + "encoding problem: %s with BOM", cookie); PyMem_Free(cookie); return _PYTOK_ENCODING_ERROR; } @@ -390,10 +393,9 @@ _PyTok_PrepareString(struct tok_state *tok, const char *input, int utf8_only, if (stored < 0) { return -1; } - tok->str = tok->source.bytes != NULL ? tok->source.bytes : (char *)""; if (!utf8_only && (tok->encoding == NULL || strcmp(tok->encoding, "utf-8") == 0) && - !_PyTokenizer_ensure_utf8(tok->str, tok, 1)) { + !_PyTokenizer_ensure_utf8(_PyTok_SourceData(&tok->source), tok, 1)) { return -1; } return 0; diff --git a/Parser/tokenizer/helpers.c b/Parser/tokenizer/helpers.c index c803b787d9dae68..0d3ea85109ec49e 100644 --- a/Parser/tokenizer/helpers.c +++ b/Parser/tokenizer/helpers.c @@ -137,10 +137,6 @@ _PyTokenizer_indenterror(struct tok_state *tok) int _PyTokenizer_warn_invalid_escape_sequence(struct tok_state *tok, int first_invalid_escape_char) { - if (!tok->report_warnings) { - return 0; - } - PyObject *msg = PyUnicode_FromFormat( "\"\\%c\" is an invalid escape sequence. " "Such sequences will not work in the future. " @@ -226,10 +222,6 @@ _PyTokenizer_raise_init_error(PyObject *filename) int _PyTokenizer_parser_warn(struct tok_state *tok, PyObject *category, const char *format, ...) { - if (!tok->report_warnings) { - return 0; - } - PyObject *errmsg; va_list vargs; va_start(vargs, format); @@ -329,26 +321,21 @@ _PyTokenizer_ensure_utf8(const char *line, struct tok_state *tok, int lineno) const char *badchar = NULL; const char *c; int length; - int col_offset = 0; const char *line_start = line; for (c = line; *c; c += length) { if (!(length = valid_utf8((const unsigned char *)c))) { badchar = c; break; } - col_offset++; if (*c == '\n') { lineno++; - col_offset = 0; line_start = c + 1; } } if (badchar) { - tok->lineno = lineno; - tok->line_start = _PyLexer_BufferOffset(tok, line_start); - tok->cur = _PyLexer_BufferOffset(tok, badchar); - _PyTokenizer_syntaxerror_known_range(tok, - col_offset + 1, col_offset + 1, + _PyTokenizer_syntaxerror_at( + tok, line_start, badchar - line_start + 1, lineno, + -1, -1, "Non-UTF-8 code starting with '\\x%.2x'" "%s%V on line %i, " "but no encoding declared; " @@ -391,12 +378,4 @@ _PyTokenizer_print_escape(FILE *f, const char *s, Py_ssize_t size) } putc('"', f); } - -void -_PyTokenizer_tok_dump(int type, char *start, char *end) -{ - fprintf(stderr, "%s", _PyParser_TokenNames[type]); - if (type == NAME || type == NUMBER || type == STRING || type == OP) - fprintf(stderr, "(%.*s)", (int)(end - start), start); -} #endif diff --git a/Parser/tokenizer/helpers.h b/Parser/tokenizer/helpers.h index 24f2d0cff1effb6..f33d8f55ed1d17c 100644 --- a/Parser/tokenizer/helpers.h +++ b/Parser/tokenizer/helpers.h @@ -5,20 +5,24 @@ #include "../lexer/state.h" -int _PyTokenizer_syntaxerror_at(struct tok_state *, const char *, - Py_ssize_t, int, int, int, const char *, ...); int _PyTokenizer_syntaxerror(struct tok_state *tok, const char *format, ...); +/* Positive range columns are 1-based byte columns. A start column of -1 + derives the character column from the reporting cursor; an end column of + -1 uses the start column. */ int _PyTokenizer_syntaxerror_known_range(struct tok_state *tok, int col_offset, int end_col_offset, const char *format, ...); +int _PyTokenizer_syntaxerror_at( + struct tok_state *tok, const char *line_start, Py_ssize_t cursor_offset, + int lineno, int col_offset, int end_col_offset, const char *format, ...); int _PyTokenizer_indenterror(struct tok_state *tok); int _PyTokenizer_warn_invalid_escape_sequence(struct tok_state *tok, int first_invalid_escape_char); int _PyTokenizer_parser_warn(struct tok_state *tok, PyObject *category, const char *format, ...); + void _PyTokenizer_raise_init_error(PyObject *filename); int _PyTokenizer_ensure_utf8(const char *line, struct tok_state *tok, int lineno); #ifdef Py_DEBUG void _PyTokenizer_print_escape(FILE *f, const char *s, Py_ssize_t size); -void _PyTokenizer_tok_dump(int type, char *start, char *end); #endif diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c index 98a464210146891..ccd2036d828724d 100644 --- a/Parser/tokenizer/reader.c +++ b/Parser/tokenizer/reader.c @@ -1,11 +1,11 @@ #include "Python.h" #include "pycore_fileutils.h" +#include "pycore_pystate.h" #include "errcode.h" #include "helpers.h" #include "reader.h" #include "reader_internal.h" -#include "../lexer/lexer.h" #include "../lexer/state.h" #ifdef HAVE_UNISTD_H @@ -136,10 +136,10 @@ chunk_is_line(const _PyTok_Chunk *chunk) static _PyTok_ReadResult next_prepared(struct tok_state *tok, _PyTok_Chunk *chunk) { - int lineno = tok->lineno + 1; - if (lineno > tok->source.nlines) { + if (tok->lineno >= tok->source.nlines) { return _PYTOK_READ_EOF; } + int lineno = tok->lineno + 1; const char *start = _PyLexer_BufferPointer(tok, tok->inp); const char *newline = memchr( start, '\n', tok->source.bytes + tok->source.len - start); @@ -205,7 +205,6 @@ initialize_file(struct tok_state *tok) if (result != _PYTOK_READ_LINE) { return -1; } - reader->prefetched_count = 1; Py_ssize_t bom_len; _PyTok_EncodingResult detection = _PyTok_DetectEncoding( tok, &reader->prefetched_lines[0], NULL, 0, &bom_len); @@ -223,16 +222,13 @@ initialize_file(struct tok_state *tok) reader->prefetched_lines[0].data = first; reader->prefetched_lines[0].ownership = _PYTOK_CHUNK_PYMEM; result = read_file_line(tok, &reader->prefetched_lines[1]); - if (result == _PYTOK_READ_LINE) { - reader->prefetched_count = 2; - } - else if (result == _PYTOK_READ_EOF) { + if (result == _PYTOK_READ_EOF) { reader->file_eof = 1; } - else { + else if (result != _PYTOK_READ_LINE) { return -1; } - _PyTok_Chunk *second = reader->prefetched_count == 2 + _PyTok_Chunk *second = reader->prefetched_lines[1].data != NULL ? &reader->prefetched_lines[1] : NULL; detection = _PyTok_DetectEncoding( tok, &reader->prefetched_lines[0], second, 1, &bom_len); @@ -302,10 +298,13 @@ next_file(struct tok_state *tok, _PyTok_Chunk *chunk) return _PYTOK_READ_LINE; } _PyTok_Chunk input = {0}; - if (reader->prefetched_index < reader->prefetched_count) { - input = reader->prefetched_lines[reader->prefetched_index]; - reader->prefetched_lines[reader->prefetched_index++] = - (_PyTok_Chunk){0}; + if (reader->prefetched_lines[0].data != NULL) { + input = reader->prefetched_lines[0]; + reader->prefetched_lines[0] = (_PyTok_Chunk){0}; + } + else if (reader->prefetched_lines[1].data != NULL) { + input = reader->prefetched_lines[1]; + reader->prefetched_lines[1] = (_PyTok_Chunk){0}; } else if (!reader->file_eof) { _PyTok_ReadResult result = read_file_line(tok, &input); @@ -474,13 +473,13 @@ static _PyTok_ReadResult next_interactive(struct tok_state *tok, _PyTok_Chunk *chunk) { _PyTok_Reader *reader = tok->reader; - if (tok->interactive_underflow == IUNDERFLOW_STOP) { + if (reader->stop_interactive) { return _PYTOK_READ_STOPPED; } char *input = PyOS_Readline( - tok->fp != NULL ? tok->fp : stdin, stdout, tok->prompt); + tok->fp != NULL ? tok->fp : stdin, stdout, reader->prompt); if (reader->nextprompt != NULL) { - tok->prompt = reader->nextprompt; + reader->prompt = reader->nextprompt; } if (input == NULL) { return _PYTOK_READ_INTERRUPT; @@ -503,7 +502,7 @@ next_interactive(struct tok_state *tok, _PyTok_Chunk *chunk) } chunk->data = _PyTok_NormalizeNewlines( decoded.data, decoded.len, 0, 0, - &chunk->len, &chunk->implicit_newline); + &chunk->len, NULL); _PyTok_ChunkClear(&decoded); if (chunk->data == NULL) { PyErr_NoMemory(); @@ -514,6 +513,32 @@ next_interactive(struct tok_state *tok, _PyTok_Chunk *chunk) return _PYTOK_READ_LINE; } +int +_PyTok_ReaderIsInteractive(const struct tok_state *tok) +{ + return tok->reader->kind == _PYTOK_READER_INTERACTIVE; +} + +const char * +_PyTokenizer_RetainedSource(const struct tok_state *tok) +{ + if (tok->reader->kind == _PYTOK_READER_PREPARED) { + return _PyTok_SourceData(&tok->source); + } + if (tok->reader->kind == _PYTOK_READER_INTERACTIVE) { + return tok->source.bytes; + } + return NULL; +} + +void +_PyTok_ReaderStopInteractive(struct tok_state *tok) +{ + if (_PyTok_ReaderIsInteractive(tok)) { + tok->reader->stop_interactive = 1; + } +} + static _PyTok_ReadResult reader_next(struct tok_state *tok, _PyTok_Chunk *chunk) { @@ -578,12 +603,13 @@ _PyTok_ReaderUnderflow(struct tok_state *tok) } return 0; } - - Py_ssize_t scan_len = chunk.len; - if (kind == _PYTOK_READER_INTERACTIVE && - chunk.implicit_newline) { - scan_len--; + if (tok->lineno == INT_MAX) { + PyErr_SetString(PyExc_OverflowError, "too many tokenizer source lines"); + tok->done = E_ERROR; + _PyTok_ChunkClear(&chunk); + return 0; } + if (!prepared) { if (streaming && reset_buffer) { reset_streaming_buffer(tok); @@ -603,11 +629,7 @@ _PyTok_ReaderUnderflow(struct tok_state *tok) tok->line_start = tok->buf_offset; tok->start = -1; } - tok->inp = source_start + scan_len; - } - if (tok->fp_interactive) { - tok->interactive_src_start = tok->source.bytes; - tok->interactive_src_end = tok->source.bytes + tok->source.len; + tok->inp = source_start + chunk.len; } if (prepared) { if (tok->start < 0 && _PyLexer_CurrentFTString(tok) == NULL) { @@ -632,10 +654,21 @@ _PyTok_ReaderUnderflow(struct tok_state *tok) static struct tok_state * tokenizer_new_with_reader(_PyTok_ReaderKind kind) { - struct tok_state *tok = _PyTokenizer_tok_new(); + struct tok_state *tok = PyMem_Calloc(1, sizeof(*tok)); if (tok == NULL) { + PyErr_NoMemory(); return NULL; } + tok->start = tok->line_start = -1; + _PyTok_SourceInit(&tok->source); + tok->done = E_OK; + tok->layout.at_bol = 1; + tok->start_loc = (_PyTok_Loc){-1, -1}; + tok->ftstring_stack = tok->ftstring_stack_inline; + tok->ftstring_capacity = FTSTRING_STACK_INLINE_CAPACITY; +#ifdef Py_DEBUG + tok->debug = _Py_GetConfig()->parser_debug; +#endif tok->reader = PyMem_Calloc(1, sizeof(*tok->reader)); if (tok->reader == NULL) { PyErr_NoMemory(); @@ -713,7 +746,7 @@ _PyTokenizer_FromFile(FILE *fp, const char *encoding, return NULL; } tok->fp = fp; - tok->prompt = ps1; + tok->reader->prompt = ps1; tok->reader->nextprompt = ps2; return tok; } @@ -767,13 +800,10 @@ _PyTokenizer_FindEncodingFilename(int fd, PyObject *filename) _PyTokenizer_Free(tok); return NULL; } - /* Reporting a warning here could recursively ask for the encoding. */ - tok->report_warnings = 0; - while (tok->lineno < 2 && tok->done == E_OK) { - struct token token; - _PyToken_Init(&token); - _PyTokenizer_Get(tok, &token); - _PyToken_Free(&token); + if (initialize_file(tok) < 0) { + fclose(fp); + _PyTokenizer_Free(tok); + return NULL; } fclose(fp); char *encoding = tok->encoding == NULL diff --git a/Parser/tokenizer/reader.h b/Parser/tokenizer/reader.h index c27bc2aa3fb8197..2913e52b9d563b4 100644 --- a/Parser/tokenizer/reader.h +++ b/Parser/tokenizer/reader.h @@ -5,5 +5,7 @@ struct tok_state; void _PyTok_ReaderFree(struct tok_state *); int _PyTok_ReaderUnderflow(struct tok_state *); +int _PyTok_ReaderIsInteractive(const struct tok_state *); +void _PyTok_ReaderStopInteractive(struct tok_state *); #endif diff --git a/Parser/tokenizer/reader_internal.h b/Parser/tokenizer/reader_internal.h index 121d0f96f6698a2..b4f6807c207a4aa 100644 --- a/Parser/tokenizer/reader_internal.h +++ b/Parser/tokenizer/reader_internal.h @@ -32,33 +32,32 @@ typedef enum { typedef struct { char *data; - Py_ssize_t len; - int implicit_newline; PyObject *owner; + Py_ssize_t len; _PyTok_ChunkOwnership ownership; + unsigned char implicit_newline; } _PyTok_Chunk; typedef struct _PyTok_Reader { - _PyTok_ReaderKind kind; PyObject *readline; PyObject *decoder; + const char *prompt; const char *nextprompt; char *file_buffer; Py_ssize_t file_buffer_cap; _PyTok_Chunk prefetched_lines[2]; - int prefetched_index; - int prefetched_count; char *decoded; Py_ssize_t decoded_pos; Py_ssize_t decoded_len; Py_ssize_t decoded_cap; - int decoded_tail_is_implicit; - - int file_initialized; - int file_eof; - int decoder_finalized; + _PyTok_ReaderKind kind; + unsigned char decoded_tail_is_implicit; + unsigned char file_initialized; + unsigned char file_eof; + unsigned char decoder_finalized; + unsigned char stop_interactive; } _PyTok_Reader; struct tok_state; diff --git a/Parser/tokenizer/source.c b/Parser/tokenizer/source.c index 2f2aaf2589246d9..d69eab93923e81c 100644 --- a/Parser/tokenizer/source.c +++ b/Parser/tokenizer/source.c @@ -2,8 +2,6 @@ #include "source.h" -#define LINE_CHECKPOINT_INTERVAL 256 - void _PyTok_SourceInit(_PyTok_SourceText *source) { @@ -14,7 +12,6 @@ void _PyTok_SourceClear(_PyTok_SourceText *source) { PyMem_Free(source->bytes); - PyMem_Free(source->line_checkpoints); PyMem_Free(source->implicit_lines); _PyTok_SourceInit(source); } @@ -74,34 +71,6 @@ reserve_bytes(_PyTok_SourceText *source, Py_ssize_t needed) return 0; } -static int -reserve_checkpoints(_PyTok_SourceText *source, int needed) -{ - if (needed <= source->checkpoints_cap) { - return 0; - } - int cap; - if (source->checkpoints_cap == 0) { - cap = 16; - } - else if (source->checkpoints_cap <= INT_MAX / 2) { - cap = source->checkpoints_cap * 2; - } - else { - PyErr_NoMemory(); - return -1; - } - _PyTok_Off *checkpoints = source->line_checkpoints; - PyMem_Resize(checkpoints, _PyTok_Off, cap); - if (checkpoints == NULL) { - PyErr_NoMemory(); - return -1; - } - source->line_checkpoints = checkpoints; - source->checkpoints_cap = cap; - return 0; -} - static int reserve_implicit_lines(_PyTok_SourceText *source, int nlines) { @@ -165,11 +134,7 @@ _PyTok_SourceAppendLine(_PyTok_SourceText *source, const char *bytes, return -1; } int nlines = source->nlines + 1; - int checkpoint = ((nlines - 1) % LINE_CHECKPOINT_INTERVAL) == 0; - int checkpoint_count = (nlines - 1) / LINE_CHECKPOINT_INTERVAL + 1; - if ((checkpoint && - reserve_checkpoints(source, checkpoint_count) < 0) || - (implicit_newline && reserve_implicit_lines(source, nlines) < 0) || + if ((implicit_newline && reserve_implicit_lines(source, nlines) < 0) || reserve_bytes(source, source->len + len + 1) < 0) { return -1; } @@ -178,10 +143,6 @@ _PyTok_SourceAppendLine(_PyTok_SourceText *source, const char *bytes, memcpy(source->bytes + start, bytes, len); source->len += len; source->bytes[source->len] = '\0'; - if (checkpoint) { - source->line_checkpoints[checkpoint_count - 1] = - source->base_offset + start; - } if (implicit_newline) { source->implicit_lines[(nlines - 1) / 8] |= (unsigned char)(1U << ((nlines - 1) & 7)); @@ -191,16 +152,23 @@ _PyTok_SourceAppendLine(_PyTok_SourceText *source, const char *bytes, } const char * -_PyTok_SourceSpanView(const _PyTok_SourceText *source, _PyTok_Span span, +_PyTok_SourceLineView(const _PyTok_SourceText *source, Py_ssize_t lineno, Py_ssize_t *len) { - if (!_PyTok_SpanIsValid(span) || span.start < source->base_offset || - span.end - source->base_offset > source->len || len == NULL) { - PyErr_SetString(PyExc_SystemError, "invalid tokenizer source span"); - return NULL; + assert(len != NULL); + const char *line = _PyTok_SourceData(source); + const char *end = line + source->len; + while (lineno > 1) { + const char *newline = memchr(line, '\n', end - line); + if (newline == NULL) { + break; + } + line = newline + 1; + lineno--; } - *len = span.end - span.start; - return _PyTok_SourceData(source) + (span.start - source->base_offset); + const char *newline = memchr(line, '\n', end - line); + *len = (newline != NULL ? newline : end) - line; + return line; } int @@ -213,124 +181,3 @@ _PyTok_SourceLineIsImplicit(const _PyTok_SourceText *source, int lineno) return (source->implicit_lines[(lineno - 1) / 8] >> ((lineno - 1) & 7)) & 1; } - -static int -source_ends_in_newline(const _PyTok_SourceText *source) -{ - return source->len > 0 && source->bytes[source->len - 1] == '\n'; -} - -static int -eof_lineno(const _PyTok_SourceText *source) -{ - if (source->nlines == 0) { - return 1; - } - return source->nlines + source_ends_in_newline(source); -} - -int -_PyTok_SourceLine(const _PyTok_SourceText *source, int lineno, - _PyTok_Line *line) -{ - if (line == NULL || lineno < 1 || lineno > eof_lineno(source)) { - PyErr_SetString(PyExc_SystemError, "invalid tokenizer source line"); - return -1; - } - if (lineno > source->nlines) { - *line = (_PyTok_Line){ - .start = source->base_offset + source->len, - .end = source->base_offset + source->len, - }; - return 0; - } - - int checkpoint = (lineno - 1) / LINE_CHECKPOINT_INTERVAL; - int current = checkpoint * LINE_CHECKPOINT_INTERVAL + 1; - _PyTok_Off start = source->line_checkpoints[checkpoint]; - while (current < lineno) { - start = _PyTok_SourceFindLineEnd(source, start); - if (start < 0) { - return -1; - } - current++; - } - _PyTok_Off end = source->base_offset + source->len; - if (lineno < source->nlines) { - end = _PyTok_SourceFindLineEnd(source, start); - if (end < 0) { - return -1; - } - } - *line = (_PyTok_Line){ - .start = start, - .end = end, - .implicit_newline = _PyTok_SourceLineIsImplicit(source, lineno), - .contains_nul = memchr( - source->bytes + (start - source->base_offset), - 0, end - start) != NULL, - }; - return 0; -} - -int -_PyTok_SourceLocation(const _PyTok_SourceText *source, _PyTok_Off offset, - _PyTok_Affinity affinity, _PyTok_Loc *loc) -{ - if (offset < source->base_offset || - offset - source->base_offset > source->len || loc == NULL || - (affinity != _PYTOK_AFFINITY_LEFT && - affinity != _PYTOK_AFFINITY_RIGHT)) { - PyErr_SetString(PyExc_SystemError, "invalid tokenizer source offset"); - return -1; - } - if (source->nlines == 0 || - (offset - source->base_offset == source->len && - source_ends_in_newline(source) && - affinity == _PYTOK_AFFINITY_RIGHT)) { - *loc = (_PyTok_Loc){eof_lineno(source), 0}; - return 0; - } - - _PyTok_Off key = offset; - if (affinity == _PYTOK_AFFINITY_LEFT && key > source->base_offset) { - key--; - } - int low = 0; - int high = (source->nlines - 1) / LINE_CHECKPOINT_INTERVAL + 1; - while (low < high) { - int middle = low + (high - low) / 2; - if (source->line_checkpoints[middle] <= key) { - low = middle + 1; - } - else { - high = middle; - } - } - int checkpoint = low - 1; - if (checkpoint < 0) { - PyErr_SetString(PyExc_SystemError, "corrupt tokenizer source line index"); - return -1; - } - int lineno = checkpoint * LINE_CHECKPOINT_INTERVAL + 1; - _PyTok_Off start = source->line_checkpoints[checkpoint]; - while (lineno < source->nlines) { - _PyTok_Off end = _PyTok_SourceFindLineEnd(source, start); - if (end < 0) { - return -1; - } - if (offset < end || - (offset == end && affinity == _PYTOK_AFFINITY_LEFT)) { - break; - } - start = end; - lineno++; - } - _PyTok_Off byte_col = offset - start; - if (byte_col > INT_MAX) { - PyErr_SetString(PyExc_OverflowError, "tokenizer column is too large"); - return -1; - } - *loc = (_PyTok_Loc){lineno, (int)byte_col}; - return 0; -} diff --git a/Parser/tokenizer/source.h b/Parser/tokenizer/source.h index 2f74ed8b1f4fab3..9576419aade1d46 100644 --- a/Parser/tokenizer/source.h +++ b/Parser/tokenizer/source.h @@ -5,28 +5,13 @@ #include "types.h" -typedef enum { - _PYTOK_AFFINITY_LEFT, - _PYTOK_AFFINITY_RIGHT, -} _PyTok_Affinity; - -/* The half-open range includes the terminating newline when present. */ -typedef struct { - _PyTok_Off start; - _PyTok_Off end; - unsigned implicit_newline : 1; - unsigned contains_nul : 1; -} _PyTok_Line; - typedef struct { char *bytes; _PyTok_Off base_offset; _PyTok_Off len; _PyTok_Off cap; - _PyTok_Off *line_checkpoints; unsigned char *implicit_lines; int nlines; - int checkpoints_cap; Py_ssize_t implicit_cap; } _PyTok_SourceText; @@ -37,9 +22,9 @@ _PyTok_SourceData(const _PyTok_SourceText *source) } PyAPI_FUNC(void) _PyTok_SourceInit(_PyTok_SourceText *); -/* Clear invalidates all cursors, spans, and views for the source. */ +/* Clear invalidates all spans and views for the source. */ PyAPI_FUNC(void) _PyTok_SourceClear(_PyTok_SourceText *); -/* Discard the retained window and invalidate its cursors, spans, and views. +/* Discard the retained window and invalidate its spans and views. Keep its allocation and advance the logical base to the end of the window. */ PyAPI_FUNC(void) _PyTok_SourceDiscard(_PyTok_SourceText *); /* Append one nonempty logical line and return its start offset. The input may @@ -49,39 +34,14 @@ PyAPI_FUNC(void) _PyTok_SourceDiscard(_PyTok_SourceText *); PyAPI_FUNC(_PyTok_Off) _PyTok_SourceAppendLine( _PyTok_SourceText *source, const char *bytes, Py_ssize_t len, int implicit_newline); -/* The returned view is invalidated by SourceAppendLine and SourceClear. */ -PyAPI_FUNC(const char *) _PyTok_SourceSpanView( - const _PyTok_SourceText *, _PyTok_Span, Py_ssize_t *); -/* Look up a 1-based line in the retained window. Empty and newline-terminated - sources have an empty virtual line at EOF. */ -PyAPI_FUNC(int) _PyTok_SourceLine( - const _PyTok_SourceText *, int, _PyTok_Line *); +/* Return borrowed bytes excluding '\n', writing the byte length to *len. + Line numbers are 1-based and clamp to the first or final line; a trailing + '\n' adds an empty final line. The view need not be NUL-terminated. + This does not set an exception. Append, discard, and clear invalidate the view. */ +PyAPI_FUNC(const char *) _PyTok_SourceLineView( + const _PyTok_SourceText *source, Py_ssize_t lineno, Py_ssize_t *len); /* Return false for invalid line numbers and the virtual EOF line. */ PyAPI_FUNC(int) _PyTok_SourceLineIsImplicit( const _PyTok_SourceText *, int); -/* At a line boundary, left affinity selects the preceding line at its end; - right affinity selects the following line at byte column zero. */ -PyAPI_FUNC(int) _PyTok_SourceLocation( - const _PyTok_SourceText *, _PyTok_Off, _PyTok_Affinity, _PyTok_Loc *); - -static inline _PyTok_Off -_PyTok_SourceFindLineEnd(const _PyTok_SourceText *source, _PyTok_Off start) -{ - if (source->bytes == NULL || start < source->base_offset || - start - source->base_offset >= source->len) { - PyErr_SetString(PyExc_SystemError, - "corrupt tokenizer source line index"); - return -1; - } - _PyTok_Off relative_start = start - source->base_offset; - const char *newline = memchr( - source->bytes + relative_start, '\n', source->len - relative_start); - if (newline == NULL) { - PyErr_SetString(PyExc_SystemError, - "corrupt tokenizer source line index"); - return -1; - } - return source->base_offset + (newline - source->bytes) + 1; -} #endif diff --git a/Parser/tokenizer/tokenizer.h b/Parser/tokenizer/tokenizer.h index e9229d120871624..82a84830d7cbfe7 100644 --- a/Parser/tokenizer/tokenizer.h +++ b/Parser/tokenizer/tokenizer.h @@ -28,8 +28,17 @@ typedef struct { int at_eof; } _PyToken_View; +/* Supplemental source context for a terminal error. location is the reporting + cursor, independent of the scanner cursor; lineno == 0 means absent. + The text span may cover multiple physical lines. */ +typedef struct { + _PyTok_Loc location; + _PyTok_Span text_span; +} _PyTokenizer_Diagnostic; + typedef struct { int status; + _PyTokenizer_Diagnostic diagnostic; _PyTok_Loc location; _PyTok_Off cursor; _PyTok_Span input_span; @@ -98,6 +107,4 @@ struct tok_state *_PyTokenizer_FromFile( An exception is set on error. */ char *_PyTokenizer_FindEncodingFilename(int, PyObject *); -#define tok_dump _Py_tok_dump - #endif /* !Py_TOKENIZER_H */ diff --git a/Tools/peg_generator/pegen/build.py b/Tools/peg_generator/pegen/build.py index 1dc33520e5387d9..ce079adafcf4c7c 100644 --- a/Tools/peg_generator/pegen/build.py +++ b/Tools/peg_generator/pegen/build.py @@ -125,6 +125,7 @@ def compile_c_extension( str(MOD_DIR.parent.parent.parent / "Python" / "Python-ast.c"), str(MOD_DIR.parent.parent.parent / "Python" / "asdl.c"), str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "lexer.c"), + str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "layout.c"), str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "number.c"), str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "state.c"), str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "string.c"),