From 36ece1656bd6b18240738105ebad421eb23142ef Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sat, 11 Jul 2026 15:07:21 +0100 Subject: [PATCH 1/3] gh-153568: Don't materialize parser token text that is never read Only tokens whose text is actually consumed get a bytes object; operators and structural tokens no longer allocate one. --- ...07-11-15-01-45.gh-issue-153568.toktext.rst | 2 + Parser/pegen.c | 43 ++++++++++++++++--- 2 files changed, 39 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-15-01-45.gh-issue-153568.toktext.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-15-01-45.gh-issue-153568.toktext.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-15-01-45.gh-issue-153568.toktext.rst new file mode 100644 index 000000000000000..36504dceb86bb48 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-15-01-45.gh-issue-153568.toktext.rst @@ -0,0 +1,2 @@ +Speed up the parser by not materializing the text of tokens whose text is +never read. diff --git a/Parser/pegen.c b/Parser/pegen.c index e709031ae781598..67dda69835b773a 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -186,6 +186,32 @@ _get_keyword_or_name_type(Parser *p, const char *text, Py_ssize_t length) return NAME; } +// Token types whose text is consumed by grammar actions or helpers, other +// than NAME-derived tokens (identifiers and keywords), which always keep +// their text: error actions may print keyword text (e.g. invalid_kwarg's +// "cannot assign to True"). For every other type the token text is never +// read again, so materializing a PyBytes for it is wasted work. +static inline int +token_needs_text(int type) +{ + switch (type) { + case NAME: + case NUMBER: + case STRING: + case FSTRING_START: + case FSTRING_MIDDLE: + case FSTRING_END: + case TSTRING_START: + case TSTRING_MIDDLE: + case TSTRING_END: + case TYPE_COMMENT: + case NOTEQUAL: // _PyPegen_check_barry_as_flufl() reads its text + return 1; + default: + return 0; + } +} + static int initialize_token(Parser *p, Token *parser_token, struct token *new_token, int token_type) { assert(parser_token != NULL); @@ -194,13 +220,18 @@ initialize_token(Parser *p, Token *parser_token, struct token *new_token, int to const char *text = _PyToken_TextView(p->tok, new_token, &length); parser_token->type = token_type == NAME ? _get_keyword_or_name_type(p, text, length) : token_type; - parser_token->bytes = PyBytes_FromStringAndSize(text, length); - if (parser_token->bytes == NULL) { - return -1; + if (token_type == NAME || token_needs_text(parser_token->type)) { + parser_token->bytes = PyBytes_FromStringAndSize(text, length); + if (parser_token->bytes == NULL) { + return -1; + } + if (_PyArena_AddPyObject(p->arena, parser_token->bytes) < 0) { + Py_DECREF(parser_token->bytes); + return -1; + } } - if (_PyArena_AddPyObject(p->arena, parser_token->bytes) < 0) { - Py_DECREF(parser_token->bytes); - return -1; + else { + parser_token->bytes = NULL; } parser_token->metadata = NULL; From 0e3cfaedbd8e6939fdfa23734e3c313b266c0b30 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sat, 11 Jul 2026 23:08:44 +0100 Subject: [PATCH 2/3] Update Parser/pegen.c MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Maurycy Pawłowski-Wieroński --- Parser/pegen.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Parser/pegen.c b/Parser/pegen.c index 67dda69835b773a..865a48ef0da1514 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -220,7 +220,7 @@ initialize_token(Parser *p, Token *parser_token, struct token *new_token, int to const char *text = _PyToken_TextView(p->tok, new_token, &length); parser_token->type = token_type == NAME ? _get_keyword_or_name_type(p, text, length) : token_type; - if (token_type == NAME || token_needs_text(parser_token->type)) { + if (token_needs_text(parser_token->type)) { parser_token->bytes = PyBytes_FromStringAndSize(text, length); if (parser_token->bytes == NULL) { return -1; From 3a71cad913c98c7d03b679f2679f033d382d098f Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Wed, 26 Aug 2026 01:21:11 +0100 Subject: [PATCH 3/3] gh-153568: Preserve keyword token text --- Parser/pegen.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Parser/pegen.c b/Parser/pegen.c index 865a48ef0da1514..67dda69835b773a 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -220,7 +220,7 @@ initialize_token(Parser *p, Token *parser_token, struct token *new_token, int to const char *text = _PyToken_TextView(p->tok, new_token, &length); parser_token->type = token_type == NAME ? _get_keyword_or_name_type(p, text, length) : token_type; - if (token_needs_text(parser_token->type)) { + if (token_type == NAME || token_needs_text(parser_token->type)) { parser_token->bytes = PyBytes_FromStringAndSize(text, length); if (parser_token->bytes == NULL) { return -1;