diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-16-53-21.gh-issue-153568.newlines.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-16-53-21.gh-issue-153568.newlines.rst new file mode 100644 index 00000000000000..deaa82c96fd3ac --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-16-53-21.gh-issue-153568.newlines.rst @@ -0,0 +1,2 @@ +Speed up the tokenizer by copying source text verbatim when it contains no +carriage returns. diff --git a/Parser/tokenizer/helpers.c b/Parser/tokenizer/helpers.c index 62b0971d418c39..238cfa001b66a6 100644 --- a/Parser/tokenizer/helpers.c +++ b/Parser/tokenizer/helpers.c @@ -265,7 +265,8 @@ char * _PyTokenizer_translate_newlines(const char *s, int exec_input, int preserve_crlf, struct tok_state *tok) { int skip_next_lf = 0; - size_t needed_length = strlen(s) + 2, final_length; + size_t input_length = strlen(s); + size_t needed_length = input_length + 2, final_length; char *buf, *current; char c = '\0'; buf = PyMem_Malloc(needed_length); @@ -274,6 +275,13 @@ _PyTokenizer_translate_newlines(const char *s, int exec_input, int preserve_crlf PyErr_NoMemory(); return NULL; } + if (memchr(s, '\r', input_length) == NULL) { + // No carriage returns: nothing to translate, copy verbatim. + memcpy(buf, s, input_length); + current = buf + input_length; + c = input_length ? s[input_length - 1] : '\0'; + goto add_final_newline; + } for (current = buf; *s; s++, current++) { c = *s; if (skip_next_lf) { @@ -290,6 +298,7 @@ _PyTokenizer_translate_newlines(const char *s, int exec_input, int preserve_crlf } *current = c; } +add_final_newline: /* If this is exec input, add a newline to the end of the string if there isn't one already. */ if (exec_input && c != '\n' && c != '\0') {