diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-15-01-45.gh-issue-153568.toktext.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-15-01-45.gh-issue-153568.toktext.rst new file mode 100644 index 00000000000000..36504dceb86bb4 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-15-01-45.gh-issue-153568.toktext.rst @@ -0,0 +1,2 @@ +Speed up the parser by not materializing the text of tokens whose text is +never read. diff --git a/Parser/pegen.c b/Parser/pegen.c index 165dcb9f995095..9e5e15fd8abade 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -178,18 +178,50 @@ _get_keyword_or_name_type(Parser *p, struct token *new_token) return NAME; } +// Token types whose text is consumed by grammar actions or helpers, other +// than NAME-derived tokens (identifiers and keywords), which always keep +// their text: error actions may print keyword text (e.g. invalid_kwarg's +// "cannot assign to True"). For every other type the token text is never +// read again, so materializing a PyBytes for it is wasted work. +static inline int +token_needs_text(int type) +{ + switch (type) { + case NAME: + case NUMBER: + case STRING: + case FSTRING_START: + case FSTRING_MIDDLE: + case FSTRING_END: + case TSTRING_START: + case TSTRING_MIDDLE: + case TSTRING_END: + case TYPE_COMMENT: + case NOTEQUAL: // _PyPegen_check_barry_as_flufl() reads its text + return 1; + default: + return 0; + } +} + static int initialize_token(Parser *p, Token *parser_token, struct token *new_token, int token_type) { assert(parser_token != NULL); parser_token->type = (token_type == NAME) ? _get_keyword_or_name_type(p, new_token) : token_type; - parser_token->bytes = PyBytes_FromStringAndSize(new_token->start, new_token->end - new_token->start); - if (parser_token->bytes == NULL) { - return -1; + if (token_needs_text(parser_token->type)) { + parser_token->bytes = PyBytes_FromStringAndSize( + new_token->start, new_token->end - new_token->start); + if (parser_token->bytes == NULL) { + return -1; + } + if (_PyArena_AddPyObject(p->arena, parser_token->bytes) < 0) { + Py_DECREF(parser_token->bytes); + return -1; + } } - if (_PyArena_AddPyObject(p->arena, parser_token->bytes) < 0) { - Py_DECREF(parser_token->bytes); - return -1; + else { + parser_token->bytes = NULL; } parser_token->metadata = NULL;