From 40279577cbdd21f796e5db11a85fd6895216e999 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sat, 11 Jul 2026 16:53:21 +0100 Subject: [PATCH] gh-153568: Skip newline translation for source without carriage returns The tokenizer copied every input byte by byte to normalize newlines, but source without a carriage return needs no translation and can be copied verbatim. --- .../2026-07-11-16-53-21.gh-issue-153568.newlines.rst | 2 ++ Parser/tokenizer/helpers.c | 11 ++++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-16-53-21.gh-issue-153568.newlines.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-16-53-21.gh-issue-153568.newlines.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-16-53-21.gh-issue-153568.newlines.rst new file mode 100644 index 00000000000000..deaa82c96fd3ac --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-16-53-21.gh-issue-153568.newlines.rst @@ -0,0 +1,2 @@ +Speed up the tokenizer by copying source text verbatim when it contains no +carriage returns. diff --git a/Parser/tokenizer/helpers.c b/Parser/tokenizer/helpers.c index 62b0971d418c39..238cfa001b66a6 100644 --- a/Parser/tokenizer/helpers.c +++ b/Parser/tokenizer/helpers.c @@ -265,7 +265,8 @@ char * _PyTokenizer_translate_newlines(const char *s, int exec_input, int preserve_crlf, struct tok_state *tok) { int skip_next_lf = 0; - size_t needed_length = strlen(s) + 2, final_length; + size_t input_length = strlen(s); + size_t needed_length = input_length + 2, final_length; char *buf, *current; char c = '\0'; buf = PyMem_Malloc(needed_length); @@ -274,6 +275,13 @@ _PyTokenizer_translate_newlines(const char *s, int exec_input, int preserve_crlf PyErr_NoMemory(); return NULL; } + if (memchr(s, '\r', input_length) == NULL) { + // No carriage returns: nothing to translate, copy verbatim. + memcpy(buf, s, input_length); + current = buf + input_length; + c = input_length ? s[input_length - 1] : '\0'; + goto add_final_newline; + } for (current = buf; *s; s++, current++) { c = *s; if (skip_next_lf) { @@ -290,6 +298,7 @@ _PyTokenizer_translate_newlines(const char *s, int exec_input, int preserve_crlf } *current = c; } +add_final_newline: /* If this is exec input, add a newline to the end of the string if there isn't one already. */ if (exec_input && c != '\n' && c != '\0') {