From e4b38373b4cfba7e306786184137d96b0efc8762 Mon Sep 17 00:00:00 2001 From: Thomas Kowalski Date: Thu, 30 Apr 2026 10:53:52 +0000 Subject: [PATCH 1/5] fix: avoid MemoryError in tokenize --- Lib/test/test_tokenize.py | 10 ++++++++++ Parser/lexer/string.c | 23 ++++++++++++++++------- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index e2db09d61f409bd..a8c48397c6ac9ca 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -3348,6 +3348,16 @@ def get_tokens(string): with self.subTest(case=case): self.assertRaises(tokenize.TokenError, get_tokens, case) + def test_tstring_multiline_bang_underflow(self): + # gh-149183: t-string with '!' across two lines used to raise + # MemoryError because last_expr_end > last_expr_size produced a + # negative length that was cast to a huge size_t. + self.assertRaises( + tokenize.TokenError, + list, + tokenize.tokenize(BytesIO(b't"{!\n!x').readline), + ) + @support.skip_wasi_stack_overflow() def test_max_indent(self): MAXINDENT = 100 diff --git a/Parser/lexer/string.c b/Parser/lexer/string.c index e546954ba5c0d82..a3b30017e68f97e 100644 --- a/Parser/lexer/string.c +++ b/Parser/lexer/string.c @@ -18,12 +18,22 @@ _PyLexer_set_ftstring_expr(struct tok_state* tok, struct token *token, char c) { } PyObject *res = NULL; + Py_ssize_t expr_len = tok_mode->last_expr_size - tok_mode->last_expr_end; + if (expr_len < 0) { + /* last_expr_end > last_expr_size: happens when '{' and the closing + delimiter span different source lines, causing the strlen-based + size tracking to underflow. Treat as a tokenizer error rather + than passing a negative length (cast to huge size_t) to malloc or + PyUnicode_DecodeUTF8. */ + return -1; + } + // Look for a # character outside of string literals int hash_detected = 0; int in_string = 0; char quote_char = 0; - for (Py_ssize_t i = 0; i < tok_mode->last_expr_size - tok_mode->last_expr_end; i++) { + for (Py_ssize_t i = 0; i < expr_len; i++) { char ch = tok_mode->last_expr_buffer[i]; // Skip escaped characters @@ -60,7 +70,7 @@ _PyLexer_set_ftstring_expr(struct tok_state* tok, struct token *token, char c) { // If we found a # character in the expression, we need to handle comments if (hash_detected) { // Allocate buffer for processed result - char *result = (char *)PyMem_Malloc((tok_mode->last_expr_size - tok_mode->last_expr_end + 1) * sizeof(char)); + char *result = (char *)PyMem_Malloc((expr_len + 1) * sizeof(char)); if (!result) { return -1; } @@ -71,7 +81,7 @@ _PyLexer_set_ftstring_expr(struct tok_state* tok, struct token *token, char c) { quote_char = 0; // Current string quote char // Process each character - while (i < tok_mode->last_expr_size - tok_mode->last_expr_end) { + while (i < expr_len) { char ch = tok_mode->last_expr_buffer[i]; // Handle string quotes @@ -87,11 +97,10 @@ _PyLexer_set_ftstring_expr(struct tok_state* tok, struct token *token, char c) { } // Skip comments else if (ch == '#' && !in_string) { - while (i < tok_mode->last_expr_size - tok_mode->last_expr_end && - tok_mode->last_expr_buffer[i] != '\n') { + while (i < expr_len && tok_mode->last_expr_buffer[i] != '\n') { i++; } - if (i < tok_mode->last_expr_size - tok_mode->last_expr_end) { + if (i < expr_len) { result[j++] = '\n'; } } @@ -108,7 +117,7 @@ _PyLexer_set_ftstring_expr(struct tok_state* tok, struct token *token, char c) { } else { res = PyUnicode_DecodeUTF8( tok_mode->last_expr_buffer, - tok_mode->last_expr_size - tok_mode->last_expr_end, + expr_len, NULL ); } From a1f06978bcc1a69f4b0657dbb515ed3604d4cd5f Mon Sep 17 00:00:00 2001 From: Thomas Kowalski Date: Mon, 11 May 2026 10:47:43 +0200 Subject: [PATCH 2/5] misc: add news entry --- .../2026-05-11-10-46-00.gh-issue-149183.aB3xQr.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-05-11-10-46-00.gh-issue-149183.aB3xQr.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-11-10-46-00.gh-issue-149183.aB3xQr.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-11-10-46-00.gh-issue-149183.aB3xQr.rst new file mode 100644 index 000000000000000..14bb6ca4080bc30 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-11-10-46-00.gh-issue-149183.aB3xQr.rst @@ -0,0 +1,2 @@ +Fix :exc:`MemoryError` in the t-string tokenizer when the opening ``{`` +and closing delimiter span different source lines. From 29bed1f6b21b02d15396bb1bdc50d8cb2203a341 Mon Sep 17 00:00:00 2001 From: Thomas Kowalski Date: Mon, 11 May 2026 10:47:48 +0200 Subject: [PATCH 3/5] review: move check before declaration --- Parser/lexer/string.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Parser/lexer/string.c b/Parser/lexer/string.c index a3b30017e68f97e..1d6d24a41d2396f 100644 --- a/Parser/lexer/string.c +++ b/Parser/lexer/string.c @@ -16,7 +16,6 @@ _PyLexer_set_ftstring_expr(struct tok_state* tok, struct token *token, char c) { if (!(tok_mode->in_debug || tok_mode->string_kind == TSTRING) || token->metadata) { return 0; } - PyObject *res = NULL; Py_ssize_t expr_len = tok_mode->last_expr_size - tok_mode->last_expr_end; if (expr_len < 0) { @@ -28,6 +27,8 @@ _PyLexer_set_ftstring_expr(struct tok_state* tok, struct token *token, char c) { return -1; } + PyObject *res = NULL; + // Look for a # character outside of string literals int hash_detected = 0; int in_string = 0; From 1937551cc14ca5976425eaf601f2298f2afe9288 Mon Sep 17 00:00:00 2001 From: Thomas Kowalski Date: Mon, 11 May 2026 13:33:20 +0200 Subject: [PATCH 4/5] review: update Lib/test/test_tokenize.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/test/test_tokenize.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index a8c48397c6ac9ca..5d72e17b8bc13c3 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -3352,11 +3352,8 @@ def test_tstring_multiline_bang_underflow(self): # gh-149183: t-string with '!' across two lines used to raise # MemoryError because last_expr_end > last_expr_size produced a # negative length that was cast to a huge size_t. - self.assertRaises( - tokenize.TokenError, - list, - tokenize.tokenize(BytesIO(b't"{!\n!x').readline), - ) + readline = BytesIO(b't"{!\n!x').readline + self.assertRaises(tokenize.TokenError, list, tokenize.tokenize(readline)) @support.skip_wasi_stack_overflow() def test_max_indent(self): From 9a0a985b16a2f6940b78c4aa16d46941e4c4d9d9 Mon Sep 17 00:00:00 2001 From: Thomas Kowalski Date: Tue, 14 Jul 2026 11:22:45 +0200 Subject: [PATCH 5/5] review: properly set error code --- Parser/lexer/string.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Parser/lexer/string.c b/Parser/lexer/string.c index 1d6d24a41d2396f..0e7e1a3f71f88f6 100644 --- a/Parser/lexer/string.c +++ b/Parser/lexer/string.c @@ -24,6 +24,7 @@ _PyLexer_set_ftstring_expr(struct tok_state* tok, struct token *token, char c) { size tracking to underflow. Treat as a tokenizer error rather than passing a negative length (cast to huge size_t) to malloc or PyUnicode_DecodeUTF8. */ + tok->done = E_TOKEN; return -1; }