From c758acc29541a8204c429c53a4708a0288840571 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sat, 11 Jul 2026 15:07:21 +0100 Subject: [PATCH 1/2] gh-153568: Don't materialize parser token text that is never read Only tokens whose text is actually consumed get a bytes object; operators and structural tokens no longer allocate one. --- ...07-11-15-01-45.gh-issue-153568.toktext.rst | 2 + Parser/pegen.c | 44 ++++++++++++++++--- 2 files changed, 40 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-15-01-45.gh-issue-153568.toktext.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-15-01-45.gh-issue-153568.toktext.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-15-01-45.gh-issue-153568.toktext.rst new file mode 100644 index 00000000000000..36504dceb86bb4 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-15-01-45.gh-issue-153568.toktext.rst @@ -0,0 +1,2 @@ +Speed up the parser by not materializing the text of tokens whose text is +never read. diff --git a/Parser/pegen.c b/Parser/pegen.c index 165dcb9f995095..0f7256409ab2fd 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -178,18 +178,50 @@ _get_keyword_or_name_type(Parser *p, struct token *new_token) return NAME; } +// Token types whose text is consumed by grammar actions or helpers, other +// than NAME-derived tokens (identifiers and keywords), which always keep +// their text: error actions may print keyword text (e.g. invalid_kwarg's +// "cannot assign to True"). For every other type the token text is never +// read again, so materializing a PyBytes for it is wasted work. +static inline int +token_needs_text(int type) +{ + switch (type) { + case NAME: + case NUMBER: + case STRING: + case FSTRING_START: + case FSTRING_MIDDLE: + case FSTRING_END: + case TSTRING_START: + case TSTRING_MIDDLE: + case TSTRING_END: + case TYPE_COMMENT: + case NOTEQUAL: // _PyPegen_check_barry_as_flufl() reads its text + return 1; + default: + return 0; + } +} + static int initialize_token(Parser *p, Token *parser_token, struct token *new_token, int token_type) { assert(parser_token != NULL); parser_token->type = (token_type == NAME) ? _get_keyword_or_name_type(p, new_token) : token_type; - parser_token->bytes = PyBytes_FromStringAndSize(new_token->start, new_token->end - new_token->start); - if (parser_token->bytes == NULL) { - return -1; + if (token_type == NAME || token_needs_text(parser_token->type)) { + parser_token->bytes = PyBytes_FromStringAndSize( + new_token->start, new_token->end - new_token->start); + if (parser_token->bytes == NULL) { + return -1; + } + if (_PyArena_AddPyObject(p->arena, parser_token->bytes) < 0) { + Py_DECREF(parser_token->bytes); + return -1; + } } - if (_PyArena_AddPyObject(p->arena, parser_token->bytes) < 0) { - Py_DECREF(parser_token->bytes); - return -1; + else { + parser_token->bytes = NULL; } parser_token->metadata = NULL; From 1e18605b31811dd433e2ced3899a9db5ddd7afc6 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sat, 11 Jul 2026 23:08:44 +0100 Subject: [PATCH 2/2] Update Parser/pegen.c MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Maurycy Pawłowski-Wieroński --- Parser/pegen.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Parser/pegen.c b/Parser/pegen.c index 0f7256409ab2fd..9e5e15fd8abade 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -209,7 +209,7 @@ initialize_token(Parser *p, Token *parser_token, struct token *new_token, int to assert(parser_token != NULL); parser_token->type = (token_type == NAME) ? _get_keyword_or_name_type(p, new_token) : token_type; - if (token_type == NAME || token_needs_text(parser_token->type)) { + if (token_needs_text(parser_token->type)) { parser_token->bytes = PyBytes_FromStringAndSize( new_token->start, new_token->end - new_token->start); if (parser_token->bytes == NULL) {