Skip to content

gh-153569: hide tokenizer state behind token and source operations #156654

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
pablogsal merged 4 commits into from
Sep 12, 2026
Merged

gh-153569: hide tokenizer state behind token and source operations #156654

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
5caac0f
gh-153569: hide tokenizer state behind token and source operations
pablogsal Sep 6, 2026
97b6803
gh-153569: keep persistent scanner positions as source offsets
pablogsal Sep 6, 2026
1d56ef5
gh-153569: include token types and owned metadata in tokenizer results
pablogsal Sep 6, 2026
aef6c9e
gh-153569: keep character scanning inline with source offsets
pablogsal Sep 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions Lib/test/test_fstring.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1891,6 +1891,9 @@ def __format__(self, format):
self.assertEqual(f"{UnchangedFormat():{r'\xFF'}}", '\\xFF')
self.assertEqual(rf"{UnchangedFormat():{r'\xFF'}}", '\\xFF')

self.assertEqual(rf"{UnchangedFormat():{f'\xFF'}}\n", 'ÿ\\n')
self.assertEqual(f"{UnchangedFormat():{rf'\xFF'}}\n", '\\xFF\n')

# Test continuation character in format specs
self.assertEqual(f"""{UnchangedFormat():{'a'\
'b'}}""", 'ab')
Expand Down
12 changes: 12 additions & 0 deletions Lib/test/test_tstring.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,14 @@ def test_raw_tstrings(self):
t = tr"{path}\Documents"
self.assertTStringEqual(t, ("", r"\Documents"), [(path, "path")])

value = 42
t = rt"{value:{f'\xFF'}}\n"
self.assertTStringEqual(
t, ("", "\\n"), [(value, "value", None, 'ÿ')])
t = t"{value:{rf'\xFF'}}\n"
self.assertTStringEqual(
t, ("", "\n"), [(value, "value", None, '\\xFF')])

def test_template_concatenation(self):
# Test template + template
t1 = t"Hello, "
Expand Down Expand Up @@ -226,6 +234,10 @@ def test_syntax_errors(self):
("t'{x=!}'", "t-string: missing conversion character"),
("t'{x!z}'", "t-string: invalid conversion character 'z': "
"expected 's', 'r', or 'a'"),
("f\"{t'{x!z}'}\"", "t-string: invalid conversion character 'z': "
"expected 's', 'r', or 'a'"),
("t'{f\"{x!z}\"}'", "f-string: invalid conversion character 'z': "
"expected 's', 'r', or 'a'"),
("t'{lambda:1}'", "t-string: lambda expressions are not allowed "
"without parentheses"),
("t'{x:{;}}'", "t-string: expecting a valid expression after '{'"),
Expand Down
6 changes: 3 additions & 3 deletions Makefile.pre.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -394,13 +394,13 @@ PEGEN_OBJS= \
Parser/peg_api.o

TOKENIZER_OBJS= \
Parser/lexer/buffer.o \
Parser/lexer/lexer.o \
Parser/lexer/number.o \
Parser/lexer/state.o \
Parser/lexer/string.o \
Parser/tokenizer/cursor.o \
Parser/tokenizer/decoder.o \
Parser/tokenizer/api.o \
Parser/tokenizer/reader.o \
Parser/tokenizer/source.o \
Parser/tokenizer/helpers.o
Expand All @@ -411,14 +411,14 @@ PEGEN_HEADERS= \
$(srcdir)/Parser/string_parser.h

TOKENIZER_HEADERS= \
Parser/lexer/buffer.h \
Parser/lexer/lexer.h \
Parser/lexer/lexer_internal.h \
Parser/lexer/state.h \
Parser/tokenizer/cursor.h \
Parser/tokenizer/reader.h \
Parser/tokenizer/reader_internal.h \
Parser/tokenizer/source.h \
Parser/tokenizer/types.h \
Parser/tokenizer/tokenizer.h \
Parser/tokenizer/helpers.h

Expand Down Expand Up @@ -3463,7 +3463,7 @@ MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo.
MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_openssl_mem.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h
MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/parts.h $(srcdir)/Modules/_testcapi/util.h
MODULE__TESTLIMITEDCAPI_DEPS=$(srcdir)/Modules/_testlimitedcapi/testcapi_long.h $(srcdir)/Modules/_testlimitedcapi/parts.h $(srcdir)/Modules/_testlimitedcapi/util.h
MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h $(srcdir)/Parser/tokenizer/cursor.h $(srcdir)/Parser/tokenizer/source.h $(srcdir)/Python/ceval.h $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h
MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h $(srcdir)/Parser/tokenizer/cursor.h $(srcdir)/Parser/tokenizer/source.h $(srcdir)/Parser/tokenizer/types.h $(srcdir)/Python/ceval.h $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h
MODULE__SQLITE3_DEPS=$(srcdir)/Modules/_sqlite/connection.h $(srcdir)/Modules/_sqlite/cursor.h $(srcdir)/Modules/_sqlite/microprotocols.h $(srcdir)/Modules/_sqlite/module.h $(srcdir)/Modules/_sqlite/prepare_protocol.h $(srcdir)/Modules/_sqlite/row.h $(srcdir)/Modules/_sqlite/util.h
MODULE__ZSTD_DEPS=$(srcdir)/Modules/_zstd/_zstdmodule.h $(srcdir)/Modules/_zstd/buffer.h $(srcdir)/Modules/_zstd/zstddict.h

Expand Down
2 changes: 1 addition & 1 deletion PCbuild/_freeze_module.vcxproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -181,12 +181,12 @@
<ClCompile Include="..\Parser\action_helpers.c" />
<ClCompile Include="..\Parser\string_parser.c" />
<ClCompile Include="..\Parser\token.c" />
<ClCompile Include="..\Parser\lexer\buffer.c" />
<ClCompile Include="..\Parser\lexer\state.c" />
<ClCompile Include="..\Parser\lexer\lexer.c" />
<ClCompile Include="..\Parser\lexer\number.c" />
<ClCompile Include="..\Parser\lexer\string.c" />
<ClCompile Include="..\Parser\tokenizer\decoder.c" />
<ClCompile Include="..\Parser\tokenizer\api.c" />
<ClCompile Include="..\Parser\tokenizer\reader.c" />
<ClCompile Include="..\Parser\tokenizer\source.c" />
<ClCompile Include="..\Parser\tokenizer\helpers.c" />
Expand Down
6 changes: 3 additions & 3 deletions PCbuild/_freeze_module.vcxproj.filters
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -469,15 +469,15 @@
<ClCompile Include="..\Parser\lexer\string.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\Parser\lexer\buffer.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\Parser\lexer\state.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\Parser\tokenizer\decoder.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\Parser\tokenizer\api.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\Parser\tokenizer\reader.c">
<Filter>Source Files</Filter>
</ClCompile>
Expand Down
4 changes: 2 additions & 2 deletions PCbuild/pythoncore.vcxproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -423,10 +423,10 @@
<ClInclude Include="..\Parser\lexer\state.h" />
<ClInclude Include="..\Parser\lexer\lexer.h" />
<ClInclude Include="..\Parser\lexer\lexer_internal.h" />
<ClInclude Include="..\Parser\lexer\buffer.h" />
<ClInclude Include="..\Parser\tokenizer\cursor.h" />
<ClInclude Include="..\Parser\tokenizer\reader.h" />
<ClInclude Include="..\Parser\tokenizer\reader_internal.h" />
<ClInclude Include="..\Parser\tokenizer\types.h" />
<ClInclude Include="..\Parser\tokenizer\source.h" />
<ClInclude Include="..\Parser\tokenizer\helpers.h" />
<ClInclude Include="..\Parser\tokenizer\tokenizer.h" />
Expand Down Expand Up @@ -593,10 +593,10 @@
<ClCompile Include="..\Parser\lexer\lexer.c" />
<ClCompile Include="..\Parser\lexer\number.c" />
<ClCompile Include="..\Parser\lexer\string.c" />
<ClCompile Include="..\Parser\lexer\buffer.c" />
<ClCompile Include="..\Parser\tokenizer\cursor.c" />
<ClCompile Include="..\Parser\tokenizer\source.c" />
<ClCompile Include="..\Parser\tokenizer\decoder.c" />
<ClCompile Include="..\Parser\tokenizer\api.c" />
<ClCompile Include="..\Parser\tokenizer\reader.c" />
<ClCompile Include="..\Parser\tokenizer\helpers.c" />
<ClCompile Include="..\Parser\token.c" />
Expand Down
12 changes: 6 additions & 6 deletions PCbuild/pythoncore.vcxproj.filters
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -330,9 +330,6 @@
<ClInclude Include="..\Parser\lexer\lexer_internal.h">
<Filter>Parser</Filter>
</ClInclude>
<ClInclude Include="..\Parser\lexer\buffer.h">
<Filter>Parser</Filter>
</ClInclude>
<ClInclude Include="..\Parser\tokenizer\cursor.h">
<Filter>Parser</Filter>
</ClInclude>
Expand All @@ -342,6 +339,9 @@
<ClInclude Include="..\Parser\tokenizer\reader_internal.h">
<Filter>Parser</Filter>
</ClInclude>
<ClInclude Include="..\Parser\tokenizer\types.h">
<Filter>Parser</Filter>
</ClInclude>
<ClInclude Include="..\Parser\tokenizer\source.h">
<Filter>Parser</Filter>
</ClInclude>
Expand Down Expand Up @@ -1361,9 +1361,6 @@
<ClCompile Include="..\Parser\lexer\state.c">
<Filter>Parser</Filter>
</ClCompile>
<ClCompile Include="..\Parser\lexer\buffer.c">
<Filter>Parser</Filter>
</ClCompile>
<ClCompile Include="..\Parser\tokenizer\cursor.c">
<Filter>Parser</Filter>
</ClCompile>
Expand All @@ -1373,6 +1370,9 @@
<ClCompile Include="..\Parser\tokenizer\decoder.c">
<Filter>Parser</Filter>
</ClCompile>
<ClCompile Include="..\Parser\tokenizer\api.c">
<Filter>Parser</Filter>
</ClCompile>
<ClCompile Include="..\Parser\tokenizer\reader.c">
<Filter>Parser</Filter>
</ClCompile>
Expand Down
34 changes: 23 additions & 11 deletions Parser/action_helpers.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1004,8 +1004,20 @@ result_token_with_metadata(Parser *p, void *result, PyObject *metadata)
static char
formatted_string_prefix(const Parser *p)
{
const ftstring_state *state = _PyLexer_CurrentFTString(p->tok);
return state == NULL ? 'f' : _PyLexer_StringPrefix(state->kind);
int nested = 0;
for (int i = p->mark - 1; i >= 0; i--) {
int type = p->tokens[i]->type;
if (type == FSTRING_END || type == TSTRING_END) {
nested++;
}
else if (type == FSTRING_START || type == TSTRING_START) {
if (nested == 0) {
return type == TSTRING_START ? 't' : 'f';
}
nested--;
}
}
Py_UNREACHABLE();
}

ResultTokenWithMetadata *
Expand Down Expand Up @@ -1352,7 +1364,7 @@ _PyPegen_decode_fstring_part(Parser* p, int is_raw, expr_ty constant, Token* tok

static asdl_expr_seq *
_get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions,
Token *b, ftstring_kind string_kind)
Token *b, int is_tstring)
{
Py_ssize_t n_items = asdl_seq_LEN(raw_expressions);
Py_ssize_t total_items = n_items;
Expand Down Expand Up @@ -1384,7 +1396,7 @@ _get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions,
asdl_expr_seq *values = item->v.JoinedStr.values;
if (asdl_seq_LEN(values) != 2) {
PyErr_Format(PyExc_SystemError,
_PyLexer_IsTString(string_kind)
is_tstring
? "unexpected TemplateStr node without debug data in t-string at line %d"
: "unexpected JoinedStr node without debug data in f-string at line %d",
item->lineno);
Expand All @@ -1396,7 +1408,7 @@ _get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions,
asdl_seq_SET(seq, index++, first);

expr_ty second = asdl_seq_GET(values, 1);
assert((_PyLexer_IsTString(string_kind) &&
assert((is_tstring &&
second->kind == Interpolation_kind) ||
second->kind == FormattedValue_kind);
asdl_seq_SET(seq, index++, second);
Expand Down Expand Up @@ -1440,7 +1452,7 @@ _get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions,
expr_ty
_PyPegen_template_str(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Token *b) {

asdl_expr_seq *resized_exprs = _get_resized_exprs(p, a, raw_expressions, b, TSTRING);
asdl_expr_seq *resized_exprs = _get_resized_exprs(p, a, raw_expressions, b, 1);
if (resized_exprs == NULL) {
return NULL;
}
Expand All @@ -1452,7 +1464,7 @@ _PyPegen_template_str(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Token
expr_ty
_PyPegen_joined_str(Parser *p, Token* a, asdl_expr_seq* raw_expressions, Token*b) {

asdl_expr_seq *resized_exprs = _get_resized_exprs(p, a, raw_expressions, b, FSTRING);
asdl_expr_seq *resized_exprs = _get_resized_exprs(p, a, raw_expressions, b, 0);
if (resized_exprs == NULL) {
return NULL;
}
Expand All @@ -1468,8 +1480,7 @@ expr_ty _PyPegen_decoded_constant_from_token(Parser* p, Token* tok) {
return NULL;
}

const ftstring_state *state = _PyLexer_CurrentFTString(p->tok);
int is_raw = state != NULL && _PyLexer_IsRawString(state->kind);
int is_raw = tok->is_raw;

PyObject* str = _PyPegen_decode_string(p, is_raw, bstr, bsize, tok);
if (str == NULL) {
Expand Down Expand Up @@ -2051,13 +2062,14 @@ _warn_relative_import_of_lazy(Parser *p, asdl_seq *dots, expr_ty module)
return -1;
}

_PyTokenizer_Info info = _PyTokenizer_GetInfo(p->tok);
int res = _PyErr_EmitSyntaxWarning(msg,
p->tok->filename,
info.filename,
module->lineno,
module->col_offset + 1,
module->end_lineno,
module->end_col_offset + 1,
p->tok->module);
info.module);
Py_DECREF(msg);
return res;
}
Expand Down
29 changes: 0 additions & 29 deletions Parser/lexer/buffer.c
View file
Open in desktop

This file was deleted.

21 changes: 0 additions & 21 deletions Parser/lexer/buffer.h
View file
Open in desktop

This file was deleted.

Loading
Loading