From 600a4457c033c1e023116d01d4bea855127edae2 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Fri, 28 Aug 2026 17:22:02 +0100 Subject: [PATCH 01/11] gh-153569: bound tokenizer input storage --- Lib/test/test_tokenize.py | 25 +++++ Parser/lexer/buffer.c | 81 ++++++-------- Parser/lexer/buffer.h | 18 ++- Parser/lexer/state.c | 2 +- Parser/lexer/state.h | 7 +- Parser/lexer/string.c | 8 +- Parser/tokenizer/reader.c | 169 +++++++++++++++++++++-------- Parser/tokenizer/reader_internal.h | 2 + 8 files changed, 208 insertions(+), 104 deletions(-) diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 7e02191db86be5a..0b5bdcdeda3c196 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -2427,6 +2427,31 @@ def test_stop_iteration_skips_encoded_readline_codec_lookup(self): (token.ENDMARKER, "", (1, 0), (1, 0), ""), ) + def test_fstring_offsets_survive_buffer_reallocation(self): + padding = " " * 9000 + expression_line = ")=:>{2}}\n" + physical_lines = [ + 'f"""\n', + "{(\n", + padding + "1\n", + expression_line, + '"""\n', + ] + source = "".join(physical_lines) + chunks = iter([ + "".join(physical_lines[:2]), + "".join(physical_lines[2:4]), + physical_lines[4], + "", + ]) + + expected = self._get_tokens(source, extra_tokens=True) + tokens = list(tokenize._generate_tokens_from_c_tokenizer( + chunks.__next__, + extra_tokens=True, + )) + self.assertEqual(tokens, expected) + def test_extra_tokens_relaxes_lexer_errors(self): cases = [ ( diff --git a/Parser/lexer/buffer.c b/Parser/lexer/buffer.c index cd6885a7d01040a..4bee1f85d47836d 100644 --- a/Parser/lexer/buffer.c +++ b/Parser/lexer/buffer.c @@ -1,62 +1,45 @@ #include "Python.h" -#include "errcode.h" - +#include "buffer.h" #include "state.h" -/* Traverse and remember all f-string buffers, in order to be able to restore - them after reallocating tok->buf */ void -_PyLexer_remember_fstring_buffers(struct tok_state *tok) +_PyLexer_SnapshotBuffer(struct tok_state *tok, const char *base, + _PyLexer_BufferSnapshot *snapshot) { - int index; - tokenizer_mode *mode; - - for (index = tok->tok_mode_stack_index; index >= 0; --index) { - mode = &(tok->tok_mode_stack[index]); + snapshot->buf = tok->buf - base; + snapshot->cur = tok->cur - tok->buf; + snapshot->inp = tok->inp - tok->buf; + snapshot->start = tok->start == NULL ? -1 : tok->start - tok->buf; + snapshot->line_start = tok->line_start == NULL + ? -1 : tok->line_start - tok->buf; + snapshot->multi_line_start = tok->multi_line_start == NULL + ? -1 : tok->multi_line_start - tok->buf; + for (int index = tok->tok_mode_stack_index; index >= 0; --index) { + tokenizer_mode *mode = &tok->tok_mode_stack[index]; mode->start_offset = mode->start == NULL ? -1 : mode->start - tok->buf; - mode->multi_line_start_offset = mode->multi_line_start == NULL ? -1 : mode->multi_line_start - tok->buf; + mode->multi_line_start_offset = mode->multi_line_start == NULL + ? -1 : mode->multi_line_start - tok->buf; } } -/* Traverse and restore all f-string buffers after reallocating tok->buf */ void -_PyLexer_restore_fstring_buffers(struct tok_state *tok) -{ - int index; - tokenizer_mode *mode; - - for (index = tok->tok_mode_stack_index; index >= 0; --index) { - mode = &(tok->tok_mode_stack[index]); - mode->start = mode->start_offset < 0 ? NULL : tok->buf + mode->start_offset; - mode->multi_line_start = mode->multi_line_start_offset < 0 ? NULL : tok->buf + mode->multi_line_start_offset; - } -} - -int -_PyLexer_tok_reserve_buf(struct tok_state *tok, Py_ssize_t size) +_PyLexer_RestoreBuffer(struct tok_state *tok, char *base, + const _PyLexer_BufferSnapshot *snapshot) { - Py_ssize_t cur = tok->cur - tok->buf; - Py_ssize_t oldsize = tok->inp - tok->buf; - Py_ssize_t newsize = oldsize + Py_MAX(size, oldsize >> 1); - if (newsize > tok->end - tok->buf) { - char *newbuf = tok->buf; - Py_ssize_t start = tok->start == NULL ? -1 : tok->start - tok->buf; - Py_ssize_t line_start = tok->start == NULL ? -1 : tok->line_start - tok->buf; - Py_ssize_t multi_line_start = tok->multi_line_start - tok->buf; - _PyLexer_remember_fstring_buffers(tok); - newbuf = (char *)PyMem_Realloc(newbuf, newsize); - if (newbuf == NULL) { - tok->done = E_NOMEM; - return 0; - } - tok->buf = newbuf; - tok->cur = tok->buf + cur; - tok->inp = tok->buf + oldsize; - tok->end = tok->buf + newsize; - tok->start = start < 0 ? NULL : tok->buf + start; - tok->line_start = line_start < 0 ? NULL : tok->buf + line_start; - tok->multi_line_start = multi_line_start < 0 ? NULL : tok->buf + multi_line_start; - _PyLexer_restore_fstring_buffers(tok); + tok->buf = base + snapshot->buf; + tok->cur = tok->buf + snapshot->cur; + tok->inp = tok->buf + snapshot->inp; + tok->start = snapshot->start < 0 + ? NULL : tok->buf + snapshot->start; + tok->line_start = snapshot->line_start < 0 + ? NULL : tok->buf + snapshot->line_start; + tok->multi_line_start = snapshot->multi_line_start < 0 + ? NULL : tok->buf + snapshot->multi_line_start; + for (int index = tok->tok_mode_stack_index; index >= 0; --index) { + tokenizer_mode *mode = &tok->tok_mode_stack[index]; + mode->start = mode->start_offset < 0 + ? NULL : tok->buf + mode->start_offset; + mode->multi_line_start = mode->multi_line_start_offset < 0 + ? NULL : tok->buf + mode->multi_line_start_offset; } - return 1; } diff --git a/Parser/lexer/buffer.h b/Parser/lexer/buffer.h index bb218162ff48453..06c54ff8da944a3 100644 --- a/Parser/lexer/buffer.h +++ b/Parser/lexer/buffer.h @@ -3,8 +3,20 @@ #include "pyport.h" -void _PyLexer_remember_fstring_buffers(struct tok_state *tok); -void _PyLexer_restore_fstring_buffers(struct tok_state *tok); -int _PyLexer_tok_reserve_buf(struct tok_state *tok, Py_ssize_t size); +struct tok_state; + +typedef struct { + Py_ssize_t buf; + Py_ssize_t cur; + Py_ssize_t inp; + Py_ssize_t start; + Py_ssize_t line_start; + Py_ssize_t multi_line_start; +} _PyLexer_BufferSnapshot; + +void _PyLexer_SnapshotBuffer( + struct tok_state *, const char *, _PyLexer_BufferSnapshot *); +void _PyLexer_RestoreBuffer( + struct tok_state *, char *, const _PyLexer_BufferSnapshot *); #endif diff --git a/Parser/lexer/state.c b/Parser/lexer/state.c index 2a6408bef927a36..7aadec99f837492 100644 --- a/Parser/lexer/state.c +++ b/Parser/lexer/state.c @@ -26,7 +26,6 @@ _PyTokenizer_tok_new(void) tok->interactive_src_start = NULL; tok->interactive_src_end = NULL; tok->start = NULL; - tok->end = NULL; tok->done = E_OK; tok->fp = NULL; tok->tabsize = TABSIZE; @@ -52,6 +51,7 @@ _PyTokenizer_tok_new(void) tok->comment_newline = 0; tok->implicit_newline = 0; _PyTok_SourceInit(&tok->source); + _PyTok_CursorInit(&tok->reader_cursor, &tok->source); tok->reader = NULL; tok->tok_mode_stack[0] = (tokenizer_mode){.kind =TOK_REGULAR_MODE, .quote='\0', .quote_size = 0, .in_debug=0}; tok->tok_mode_stack_index = 0; diff --git a/Parser/lexer/state.h b/Parser/lexer/state.h index 0824785195491ee..ae942a45a0ec2d4 100644 --- a/Parser/lexer/state.h +++ b/Parser/lexer/state.h @@ -2,7 +2,7 @@ #define _PY_LEXER_H_ #include "object.h" -#include "../tokenizer/source.h" +#include "../tokenizer/cursor.h" #define MAXINDENT 100 /* Max indentation level */ #define MAXLEVEL 200 /* Max parentheses level */ @@ -67,15 +67,15 @@ typedef struct _tokenizer_mode { /* Tokenizer state */ struct tok_state { - /* Input state; buf <= cur <= inp <= end */ + /* Input state; buf <= cur <= inp */ /* NB an entire line is held in the buffer */ char *buf; /* Input buffer, or NULL; malloc'ed if fp != NULL or readline != NULL */ char *cur; /* Next character in buffer */ char *inp; /* End of data in buffer */ + _PyTok_Off buf_offset; /* Logical offset of buf[0]. */ int fp_interactive; /* If the file descriptor is interactive */ char *interactive_src_start; /* The start of the source parsed so far in interactive mode */ char *interactive_src_end; /* The end of the source parsed so far in interactive mode */ - const char *end; /* End of input buffer if buf != NULL */ const char *start; /* Start of current token if not NULL */ int done; /* E_OK normally, E_EOF at EOF, otherwise error code */ /* NB If done != E_OK, cur must be == inp!!! */ @@ -110,6 +110,7 @@ struct tok_state { char* str; /* Source string being tokenized (if tokenizing from a string)*/ _PyTok_SourceText source; + _PyTok_Cursor reader_cursor; struct _PyTok_Reader *reader; int type_comments; /* Whether to look for type comments */ diff --git a/Parser/lexer/string.c b/Parser/lexer/string.c index d67c48f7f678eda..fc0299c5c7c592f 100644 --- a/Parser/lexer/string.c +++ b/Parser/lexer/string.c @@ -125,7 +125,8 @@ _PyLexer_update_ftstring_expr(struct tok_state *tok, char cur) { assert(tok->cur != NULL); - Py_ssize_t size = strlen(tok->cur); + Py_ssize_t size = cur == 0 + ? tok->inp - tok->cur : (Py_ssize_t)strlen(tok->cur); tokenizer_mode *tok_mode = TOK_GET_MODE(tok); switch (cur) { @@ -142,7 +143,8 @@ _PyLexer_update_ftstring_expr(struct tok_state *tok, char cur) goto error; } tok_mode->last_expr_buffer = new_buffer; - strncpy(tok_mode->last_expr_buffer + tok_mode->last_expr_size, tok->cur, size); + memcpy(tok_mode->last_expr_buffer + tok_mode->last_expr_size, + tok->cur, size); tok_mode->last_expr_size += size; break; case '{': @@ -155,7 +157,7 @@ _PyLexer_update_ftstring_expr(struct tok_state *tok, char cur) } tok_mode->last_expr_size = size; tok_mode->last_expr_end = -1; - strncpy(tok_mode->last_expr_buffer, tok->cur, size); + memcpy(tok_mode->last_expr_buffer, tok->cur, size); break; case '}': case '!': diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c index 82b824f56374fcf..408bfda147b4e94 100644 --- a/Parser/tokenizer/reader.c +++ b/Parser/tokenizer/reader.c @@ -13,6 +13,12 @@ # include #endif +static inline int +reader_is_streaming(_PyTok_ReaderKind kind) +{ + return kind == _PYTOK_READER_FILE || kind == _PYTOK_READER_READLINE; +} + void _PyTok_ReaderFree(struct tok_state *tok) { @@ -28,14 +34,27 @@ _PyTok_ReaderFree(struct tok_state *tok) } PyMem_Free(reader->file_buffer); PyMem_Free(reader->decoded); - if (reader->kind != _PYTOK_READER_PREPARED) { + if (reader_is_streaming(reader->kind)) { PyMem_Free(tok->buf); - tok->buf = NULL; } + tok->buf = NULL; PyMem_Free(reader); tok->reader = NULL; } +static int +resize_buffer(char **buffer, Py_ssize_t *capacity, Py_ssize_t new_capacity) +{ + char *resized = PyMem_Realloc(*buffer, new_capacity); + if (resized == NULL) { + PyErr_NoMemory(); + return -1; + } + *buffer = resized; + *capacity = new_capacity; + return 0; +} + static int reserve_buffer(char **buffer, Py_ssize_t *capacity, Py_ssize_t needed) { @@ -50,13 +69,31 @@ reserve_buffer(char **buffer, Py_ssize_t *capacity, Py_ssize_t needed) } cap *= 2; } - char *resized = PyMem_Realloc(*buffer, cap); - if (resized == NULL) { - PyErr_NoMemory(); + return resize_buffer(buffer, capacity, cap); +} + +static int +reserve_input_buffer(struct tok_state *tok, Py_ssize_t needed) +{ + _PyTok_Reader *reader = tok->reader; + if (needed <= reader->input_buffer_cap) { + return 0; + } + assert(tok->buf != NULL); + assert(tok->cur >= tok->buf && tok->cur <= tok->inp); + assert(tok->inp - tok->buf <= reader->input_buffer_cap); + Py_ssize_t used = tok->inp - tok->buf; + Py_ssize_t growth = used >> 1; + Py_ssize_t capacity = used > PY_SSIZE_T_MAX - growth + ? needed : Py_MAX(needed, used + growth); + _PyLexer_BufferSnapshot snapshot; + _PyLexer_SnapshotBuffer(tok, tok->buf, &snapshot); + char *buffer = tok->buf; + if (resize_buffer( + &buffer, &reader->input_buffer_cap, capacity) < 0) { return -1; } - *buffer = resized; - *capacity = cap; + _PyLexer_RestoreBuffer(tok, buffer, &snapshot); return 0; } @@ -529,19 +566,31 @@ reader_next(struct tok_state *tok, _PyTok_Chunk *chunk) Py_UNREACHABLE(); } +static void +reset_streaming_buffer(struct tok_state *tok) +{ + assert(tok->buf != NULL); + assert(tok->cur >= tok->buf && tok->cur <= tok->inp); + Py_ssize_t consumed = tok->inp - tok->buf; + assert(tok->buf_offset <= PY_SSIZE_T_MAX - consumed); + tok->buf_offset += consumed; + tok->cur = tok->inp = tok->buf; +} + int _PyTok_ReaderUnderflow(struct tok_state *tok) { - int prepared = tok->reader->kind == _PYTOK_READER_PREPARED; + _PyTok_ReaderKind kind = tok->reader->kind; + int prepared = kind == _PYTOK_READER_PREPARED; + int streaming = reader_is_streaming(kind); int reset_buffer = !prepared && tok->start == NULL && !INSIDE_FSTRING(tok); - if (reset_buffer && tok->reader->kind != _PYTOK_READER_INTERACTIVE) { - tok->cur = tok->inp = tok->buf; - } - _PyTok_Chunk chunk; _PyTok_ReadResult result = reader_next(tok, &chunk); if (result != _PYTOK_READ_LINE) { + if (reset_buffer && streaming) { + reset_streaming_buffer(tok); + } if (result == _PYTOK_READ_EOF) { tok->done = E_EOF; } @@ -558,34 +607,68 @@ _PyTok_ReaderUnderflow(struct tok_state *tok) ? E_NOMEM : E_ERROR; } } - if (tok->reader->kind == _PYTOK_READER_INTERACTIVE && + if (kind == _PYTOK_READER_INTERACTIVE && result != _PYTOK_READ_STOPPED) { PySys_WriteStderr("\n"); } return 0; } - Py_ssize_t copy_len = chunk.len; - if (tok->reader->kind == _PYTOK_READER_INTERACTIVE && + Py_ssize_t scan_len = chunk.len; + if (kind == _PYTOK_READER_INTERACTIVE && chunk.implicit_newline) { - copy_len--; - } - if (reset_buffer && tok->reader->kind == _PYTOK_READER_INTERACTIVE) { - tok->cur = tok->inp = tok->buf; + scan_len--; } - if (!prepared && !_PyLexer_tok_reserve_buf(tok, copy_len + 1)) { - _PyTok_ChunkClear(&chunk); - tok->input_error = 1; - return 0; + if (streaming) { + if (reset_buffer) { + reset_streaming_buffer(tok); + } + Py_ssize_t used = tok->inp - tok->buf; + int overflow = scan_len > PY_SSIZE_T_MAX - used - 1 || + tok->buf_offset > PY_SSIZE_T_MAX - used - scan_len; + if (overflow) { + PyErr_NoMemory(); + } + if (overflow || reserve_input_buffer(tok, used + scan_len + 1) < 0) { + _PyTok_ChunkClear(&chunk); + tok->done = E_NOMEM; + tok->input_error = 1; + return 0; + } + memcpy(tok->inp, chunk.data, (size_t)scan_len); + tok->inp += scan_len; + *tok->inp = '\0'; } - if (tok->reader->kind == _PYTOK_READER_INTERACTIVE && - _PyTok_SourceAppendLine(&tok->source, chunk.data, chunk.len, - chunk.implicit_newline) < 0) { - _PyTok_ChunkClear(&chunk); - tok->done = PyErr_ExceptionMatches(PyExc_MemoryError) - ? E_NOMEM : E_ERROR; - tok->input_error = 1; - return 0; + else if (!prepared) { + int source_will_grow = + chunk.len > tok->source.cap - tok->source.len - 1; + _PyLexer_BufferSnapshot snapshot; + if (!reset_buffer && source_will_grow) { + _PyLexer_SnapshotBuffer( + tok, tok->source.bytes, &snapshot); + } + _PyTok_Off source_start = _PyTok_SourceAppendLine( + &tok->source, chunk.data, chunk.len, + chunk.implicit_newline); + if (source_start < 0) { + _PyTok_ChunkClear(&chunk); + tok->done = PyErr_ExceptionMatches(PyExc_MemoryError) + ? E_NOMEM : E_ERROR; + tok->input_error = 1; + return 0; + } + if (reset_buffer) { + tok->buf = tok->cur = tok->source.bytes + source_start; + tok->buf_offset = source_start; + tok->line_start = tok->buf; + tok->start = NULL; + tok->multi_line_start = NULL; + } + else if (source_will_grow) { + _PyLexer_RestoreBuffer( + tok, tok->source.bytes, &snapshot); + } + tok->inp = tok->source.bytes + source_start + scan_len; } if (tok->fp_interactive) { tok->interactive_src_start = tok->source.bytes; @@ -594,14 +677,10 @@ _PyTok_ReaderUnderflow(struct tok_state *tok) if (prepared) { if (tok->start == NULL) { tok->buf = tok->cur; + tok->buf_offset = chunk.data - tok->source.bytes; } tok->inp = chunk.data + chunk.len; } - else { - memcpy(tok->inp, chunk.data, (size_t)copy_len); - tok->inp += copy_len; - *tok->inp = '\0'; - } tok->implicit_newline = chunk.implicit_newline; if (!prepared && tok->tok_mode_stack_index && @@ -611,7 +690,7 @@ _PyTok_ReaderUnderflow(struct tok_state *tok) return 0; } ADVANCE_LINENO(); - if (tok->reader->kind == _PYTOK_READER_FILE && + if (kind == _PYTOK_READER_FILE && (tok->encoding == NULL || strcmp(tok->encoding, "utf-8") == 0) && !_PyTokenizer_ensure_utf8(tok->cur, tok, tok->lineno)) { _PyTok_ChunkClear(&chunk); @@ -639,14 +718,15 @@ tokenizer_new_with_reader(_PyTok_ReaderKind kind) if (kind == _PYTOK_READER_PREPARED) { return tok; } - tok->buf = PyMem_Malloc(BUFSIZ); - if (tok->buf == NULL) { - PyErr_NoMemory(); - _PyTokenizer_Free(tok); - return NULL; + if (reader_is_streaming(kind)) { + if (reserve_buffer( + &tok->buf, &tok->reader->input_buffer_cap, BUFSIZ) < 0) { + _PyTokenizer_Free(tok); + return NULL; + } + tok->cur = tok->inp = tok->buf; + tok->buf[0] = '\0'; } - tok->cur = tok->inp = tok->buf; - tok->end = tok->buf + BUFSIZ; return tok; } @@ -664,7 +744,6 @@ tokenizer_from_string(const char *input, int utf8_only, int exec_input, return NULL; } tok->buf = tok->cur = tok->inp = tok->str; - tok->end = tok->buf; return tok; } diff --git a/Parser/tokenizer/reader_internal.h b/Parser/tokenizer/reader_internal.h index 121d0f96f6698a2..49a6f04ec60af27 100644 --- a/Parser/tokenizer/reader_internal.h +++ b/Parser/tokenizer/reader_internal.h @@ -44,6 +44,8 @@ typedef struct _PyTok_Reader { PyObject *decoder; const char *nextprompt; + Py_ssize_t input_buffer_cap; + char *file_buffer; Py_ssize_t file_buffer_cap; _PyTok_Chunk prefetched_lines[2]; From c10fd829f14f264952b19e55fceba378cd9e70ea Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Fri, 28 Aug 2026 18:17:35 +0100 Subject: [PATCH 02/11] gh-153569: return tokenizer tokens as source spans --- Parser/lexer/lexer.c | 17 +++++++++++-- Parser/lexer/lexer.h | 19 ++++++++++++++ Parser/lexer/state.c | 52 +++++++++++++++++++++------------------ Parser/lexer/state.h | 12 ++++----- Parser/pegen.c | 41 ++++++++++++++++-------------- Parser/tokenizer/source.h | 3 ++- Python/Python-tokenize.c | 40 +++++++++++++++++------------- 7 files changed, 115 insertions(+), 69 deletions(-) diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index a96362c8961023a..16f0a7b783f4b48 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -12,8 +12,21 @@ #define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end) -#define MAKE_TYPE_COMMENT_TOKEN(token_type, col_offset, end_col_offset) (\ - _PyLexer_type_comment_token_setup(tok, token, token_type, col_offset, end_col_offset, p_start, p_end)) + +static int +type_comment_token_setup(struct tok_state *tok, struct token *token, int type, + int col_offset, int end_col_offset, + const char *start, const char *end) +{ + _PyLexer_token_setup(tok, token, type, start, end); + token->start_loc = (_PyTok_Loc){tok->lineno, col_offset}; + token->end_loc = (_PyTok_Loc){tok->lineno, end_col_offset}; + return type; +} + +#define MAKE_TYPE_COMMENT_TOKEN(token_type, col_offset, end_col_offset) \ + type_comment_token_setup(tok, token, token_type, col_offset, \ + end_col_offset, p_start, p_end) /* Spaces in this constant are treated as "zero or more spaces or tabs" when tokenizing. */ diff --git a/Parser/lexer/lexer.h b/Parser/lexer/lexer.h index 1d97ac57b745b09..232dbaa287470eb 100644 --- a/Parser/lexer/lexer.h +++ b/Parser/lexer/lexer.h @@ -7,4 +7,23 @@ int _PyLexer_update_ftstring_expr(struct tok_state *tok, char cur); int _PyTokenizer_Get(struct tok_state *, struct token *); +/* The view points into the current input window. The next + _PyTokenizer_Get() call may discard it. */ +static inline const char * +_PyToken_TextView(const struct tok_state *tok, const struct token *token, + Py_ssize_t *length) +{ + assert(length != NULL); + if (!_PyTok_SpanIsValid(token->span)) { + *length = 0; + return ""; + } + assert(tok->buf != NULL); + assert(tok->inp >= tok->buf); + assert(token->span.start >= tok->buf_offset); + assert(token->span.end - tok->buf_offset <= tok->inp - tok->buf); + *length = token->span.end - token->span.start; + return tok->buf + (token->span.start - tok->buf_offset); +} + #endif diff --git a/Parser/lexer/state.c b/Parser/lexer/state.c index 7aadec99f837492..d82a7d0f296bac0 100644 --- a/Parser/lexer/state.c +++ b/Parser/lexer/state.c @@ -51,7 +51,6 @@ _PyTokenizer_tok_new(void) tok->comment_newline = 0; tok->implicit_newline = 0; _PyTok_SourceInit(&tok->source); - _PyTok_CursorInit(&tok->reader_cursor, &tok->source); tok->reader = NULL; tok->tok_mode_stack[0] = (tokenizer_mode){.kind =TOK_REGULAR_MODE, .quote='\0', .quote_size = 0, .in_debug=0}; tok->tok_mode_stack_index = 0; @@ -101,41 +100,46 @@ _PyToken_Free(struct token *token) { void _PyToken_Init(struct token *token) { +#ifdef Py_DEBUG + token->span = (_PyTok_Span){-1, -1}; + token->start_loc = (_PyTok_Loc){-1, -1}; + token->end_loc = (_PyTok_Loc){-1, -1}; +#endif token->metadata = NULL; } -int -_PyLexer_type_comment_token_setup(struct tok_state *tok, struct token *token, int type, int col_offset, - int end_col_offset, const char *start, const char *end) +static inline _PyTok_Span +buffer_span(const struct tok_state *tok, const char *start, const char *end) { - token->level = tok->level; - token->lineno = token->end_lineno = tok->lineno; - token->col_offset = col_offset; - token->end_col_offset = end_col_offset; - token->start = start; - token->end = end; - return type; + if (start == NULL) { + assert(end == NULL); + return (_PyTok_Span){-1, -1}; + } + assert(end != NULL); + const char *base = tok->buf; + assert(base != NULL); + assert(tok->inp >= base); + Py_ssize_t start_offset = start - base; + Py_ssize_t end_offset = end - base; + assert(start_offset >= 0 && start_offset <= end_offset); + assert(end_offset <= tok->inp - base); + assert(tok->buf_offset <= PY_SSIZE_T_MAX - end_offset); + return _PyTok_SpanFromBounds( + tok->buf_offset + start_offset, tok->buf_offset + end_offset); } int _PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, const char *start, const char *end) { - assert((start == NULL && end == NULL) || (start != NULL && end != NULL)); token->level = tok->level; - if (ISSTRINGLIT(type)) { - token->lineno = tok->first_lineno; - } - else { - token->lineno = tok->lineno; - } - token->end_lineno = tok->lineno; - token->col_offset = token->end_col_offset = -1; - token->start = start; - token->end = end; + token->span = buffer_span(tok, start, end); + int lineno = ISSTRINGLIT(type) ? tok->first_lineno : tok->lineno; + token->start_loc = (_PyTok_Loc){lineno, -1}; + token->end_loc = (_PyTok_Loc){tok->lineno, -1}; if (start != NULL && end != NULL) { - token->col_offset = tok->starting_col_offset; - token->end_col_offset = tok->col_offset; + token->start_loc.byte_col = tok->starting_col_offset; + token->end_loc.byte_col = tok->col_offset; } return type; } diff --git a/Parser/lexer/state.h b/Parser/lexer/state.h index ae942a45a0ec2d4..496962fd0484f03 100644 --- a/Parser/lexer/state.h +++ b/Parser/lexer/state.h @@ -2,7 +2,7 @@ #define _PY_LEXER_H_ #include "object.h" -#include "../tokenizer/cursor.h" +#include "../tokenizer/source.h" #define MAXINDENT 100 /* Max indentation level */ #define MAXLEVEL 200 /* Max parentheses level */ @@ -23,8 +23,9 @@ enum interactive_underflow_t { struct token { int level; - int lineno, col_offset, end_lineno, end_col_offset; - const char *start, *end; + _PyTok_Span span; + _PyTok_Loc start_loc; + _PyTok_Loc end_loc; PyObject *metadata; }; @@ -69,7 +70,7 @@ typedef struct _tokenizer_mode { struct tok_state { /* Input state; buf <= cur <= inp */ /* NB an entire line is held in the buffer */ - char *buf; /* Input buffer, or NULL; malloc'ed if fp != NULL or readline != NULL */ + char *buf; /* Owned for file/readline input; source-backed otherwise. */ char *cur; /* Next character in buffer */ char *inp; /* End of data in buffer */ _PyTok_Off buf_offset; /* Logical offset of buf[0]. */ @@ -110,7 +111,6 @@ struct tok_state { char* str; /* Source string being tokenized (if tokenizing from a string)*/ _PyTok_SourceText source; - _PyTok_Cursor reader_cursor; struct _PyTok_Reader *reader; int type_comments; /* Whether to look for type comments */ @@ -129,8 +129,6 @@ struct tok_state { #endif }; -int _PyLexer_type_comment_token_setup(struct tok_state *tok, struct token *token, int type, int col_offset, - int end_col_offset, const char *start, const char *end); int _PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, const char *start, const char *end); struct tok_state *_PyTokenizer_tok_new(void); diff --git a/Parser/pegen.c b/Parser/pegen.c index fcec810037e98d4..d86dd22444e6a7b 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -171,18 +171,17 @@ growable_comment_array_deallocate(growable_comment_array *arr) { } static int -_get_keyword_or_name_type(Parser *p, struct token *new_token) +_get_keyword_or_name_type(Parser *p, const char *text, Py_ssize_t length) { - Py_ssize_t name_len = new_token->end_col_offset - new_token->col_offset; - assert(name_len > 0); + assert(length > 0); - if (name_len >= p->n_keyword_lists || - p->keywords[name_len] == NULL || - p->keywords[name_len]->type == -1) { + if (length >= p->n_keyword_lists || + p->keywords[length] == NULL || + p->keywords[length]->type == -1) { return NAME; } - for (KeywordToken *k = p->keywords[name_len]; k != NULL && k->type != -1; k++) { - if (strncmp(k->str, new_token->start, (size_t)name_len) == 0) { + for (KeywordToken *k = p->keywords[length]; k != NULL && k->type != -1; k++) { + if (memcmp(k->str, text, (size_t)length) == 0) { return k->type; } } @@ -193,8 +192,11 @@ static int initialize_token(Parser *p, Token *parser_token, struct token *new_token, int token_type) { assert(parser_token != NULL); - parser_token->type = (token_type == NAME) ? _get_keyword_or_name_type(p, new_token) : token_type; - parser_token->bytes = PyBytes_FromStringAndSize(new_token->start, new_token->end - new_token->start); + Py_ssize_t length; + const char *text = _PyToken_TextView(p->tok, new_token, &length); + parser_token->type = token_type == NAME + ? _get_keyword_or_name_type(p, text, length) : token_type; + parser_token->bytes = PyBytes_FromStringAndSize(text, length); if (parser_token->bytes == NULL) { return -1; } @@ -214,12 +216,14 @@ initialize_token(Parser *p, Token *parser_token, struct token *new_token, int to } parser_token->level = new_token->level; - parser_token->lineno = new_token->lineno; - parser_token->col_offset = p->tok->lineno == p->starting_lineno ? p->starting_col_offset + new_token->col_offset - : new_token->col_offset; - parser_token->end_lineno = new_token->end_lineno; - parser_token->end_col_offset = p->tok->lineno == p->starting_lineno ? p->starting_col_offset + new_token->end_col_offset - : new_token->end_col_offset; + parser_token->lineno = new_token->start_loc.lineno; + parser_token->col_offset = p->tok->lineno == p->starting_lineno + ? p->starting_col_offset + new_token->start_loc.byte_col + : new_token->start_loc.byte_col; + parser_token->end_lineno = new_token->end_loc.lineno; + parser_token->end_col_offset = p->tok->lineno == p->starting_lineno + ? p->starting_col_offset + new_token->end_loc.byte_col + : new_token->end_loc.byte_col; p->fill += 1; @@ -261,13 +265,14 @@ _PyPegen_fill_token(Parser *p) // Record and skip '# type: ignore' comments while (type == TYPE_IGNORE) { - Py_ssize_t len = new_token.end_col_offset - new_token.col_offset; + Py_ssize_t len; + const char *text = _PyToken_TextView(p->tok, &new_token, &len); char *tag = PyMem_Malloc((size_t)len + 1); if (tag == NULL) { PyErr_NoMemory(); goto error; } - strncpy(tag, new_token.start, (size_t)len); + memcpy(tag, text, (size_t)len); tag[len] = '\0'; // Ownership of tag passes to the growable array if (!growable_comment_array_add(&p->type_ignore_comments, p->tok->lineno, tag)) { diff --git a/Parser/tokenizer/source.h b/Parser/tokenizer/source.h index b42ecda1b31aa50..363475ff9015e35 100644 --- a/Parser/tokenizer/source.h +++ b/Parser/tokenizer/source.h @@ -5,7 +5,8 @@ typedef Py_ssize_t _PyTok_Off; -/* Half-open byte offsets into a _PyTok_SourceText. */ +/* Spans use half-open logical byte offsets into decoded input. Their backing + storage may retain only the current input window. */ typedef struct { _PyTok_Off start; _PyTok_Off end; diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index 762b7b3e4c8d71d..eee1fc86ded781f 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -203,14 +203,15 @@ _get_current_line(tokenizeriterobject *it, const char *line_start, Py_ssize_t si } static void -_get_col_offsets(tokenizeriterobject *it, struct token token, const char *line_start, - PyObject *line, int line_changed, Py_ssize_t lineno, Py_ssize_t end_lineno, +_get_col_offsets(tokenizeriterobject *it, const char *token_start, + const char *token_end, const char *line_start, PyObject *line, + int line_changed, Py_ssize_t lineno, Py_ssize_t end_lineno, Py_ssize_t *col_offset, Py_ssize_t *end_col_offset) { _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(it); Py_ssize_t byte_offset = -1; - if (token.start != NULL && token.start >= line_start) { - byte_offset = token.start - line_start; + if (token_start != NULL && token_start >= line_start) { + byte_offset = token_start - line_start; if (line_changed) { *col_offset = _PyPegen_byte_offset_to_character_offset_line(line, 0, byte_offset); it->byte_col_offset_diff = byte_offset - *col_offset; @@ -220,15 +221,13 @@ _get_col_offsets(tokenizeriterobject *it, struct token token, const char *line_s } } - if (token.end != NULL && token.end >= it->tok->line_start) { - Py_ssize_t end_byte_offset = token.end - it->tok->line_start; + if (token_end != NULL && token_end >= it->tok->line_start) { + Py_ssize_t end_byte_offset = token_end - it->tok->line_start; if (lineno == end_lineno) { - // If the whole token is at the same line, we can just use the token.start - // buffer for figuring out the new column offset, since using line is not - // performant for very long lines. + // Avoid rescanning the prefix of a very long line. Py_ssize_t token_col_offset = _PyPegen_byte_offset_to_character_offset_line(line, byte_offset, end_byte_offset); *end_col_offset = *col_offset + token_col_offset; - it->byte_col_offset_diff += token.end - token.start - token_col_offset; + it->byte_col_offset_diff += token_end - token_start - token_col_offset; } else { *end_col_offset = _PyPegen_byte_offset_to_character_offset_raw(it->tok->line_start, end_byte_offset); @@ -263,12 +262,18 @@ tokenizeriter_next(PyObject *op) it->done = 1; goto exit; } - PyObject *str = NULL; - if (token.start == NULL || token.end == NULL) { + const char *token_start = NULL; + const char *token_end = NULL; + PyObject *str; + if (!_PyTok_SpanIsValid(token.span)) { str = Py_GetConstant(Py_CONSTANT_EMPTY_STR); } else { - str = PyUnicode_FromStringAndSize(token.start, token.end - token.start); + Py_ssize_t token_length; + token_start = _PyToken_TextView( + it->tok, &token, &token_length); + token_end = token_start + token_length; + str = PyUnicode_FromStringAndSize(token_start, token_length); } if (str == NULL) { goto exit; @@ -297,11 +302,11 @@ tokenizeriter_next(PyObject *op) goto exit; } - Py_ssize_t lineno = ISSTRINGLIT(type) ? it->tok->first_lineno : it->tok->lineno; - Py_ssize_t end_lineno = it->tok->lineno; + Py_ssize_t lineno = token.start_loc.lineno; + Py_ssize_t end_lineno = token.end_loc.lineno; Py_ssize_t col_offset = -1; Py_ssize_t end_col_offset = -1; - _get_col_offsets(it, token, line_start, line, line_changed, + _get_col_offsets(it, token_start, token_end, line_start, line, line_changed, lineno, end_lineno, &col_offset, &end_col_offset); if (it->tok->tok_extra_tokens) { @@ -317,7 +322,8 @@ tokenizeriter_next(PyObject *op) else if (type == NEWLINE) { Py_DECREF(str); if (!it->tok->implicit_newline) { - if (it->tok->start[0] == '\r') { + assert(token_start != NULL); + if (token_start[0] == '\r') { str = PyUnicode_FromString("\r\n"); } else { str = PyUnicode_FromString("\n"); From 8c4469b09b59e2546c172190777a07eda7b954ca Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Fri, 28 Aug 2026 23:31:50 +0100 Subject: [PATCH 03/11] gh-153569: simplify tokenizer buffer relocation --- Parser/lexer/buffer.c | 43 ++++++++++++++++++------------------ Parser/lexer/buffer.h | 22 +++++++++---------- Parser/tokenizer/reader.c | 46 +++++++++++++++------------------------ Python/Python-tokenize.c | 16 ++++++++------ 4 files changed, 60 insertions(+), 67 deletions(-) diff --git a/Parser/lexer/buffer.c b/Parser/lexer/buffer.c index 4bee1f85d47836d..9c39544ca7c4790 100644 --- a/Parser/lexer/buffer.c +++ b/Parser/lexer/buffer.c @@ -3,18 +3,19 @@ #include "state.h" void -_PyLexer_SnapshotBuffer(struct tok_state *tok, const char *base, - _PyLexer_BufferSnapshot *snapshot) +_PyLexer_SaveBufferPointers(struct tok_state *tok, const char *base, + _PyLexer_BufferPointers *pointers) { - snapshot->buf = tok->buf - base; - snapshot->cur = tok->cur - tok->buf; - snapshot->inp = tok->inp - tok->buf; - snapshot->start = tok->start == NULL ? -1 : tok->start - tok->buf; - snapshot->line_start = tok->line_start == NULL + pointers->buf_from_base = tok->buf - base; + pointers->cur_from_buf = tok->cur - tok->buf; + pointers->inp_from_buf = tok->inp - tok->buf; + pointers->start_from_buf = tok->start == NULL + ? -1 : tok->start - tok->buf; + pointers->line_start_from_buf = tok->line_start == NULL ? -1 : tok->line_start - tok->buf; - snapshot->multi_line_start = tok->multi_line_start == NULL + pointers->multi_line_start_from_buf = tok->multi_line_start == NULL ? -1 : tok->multi_line_start - tok->buf; - for (int index = tok->tok_mode_stack_index; index >= 0; --index) { + for (int index = tok->tok_mode_stack_index; index > 0; --index) { tokenizer_mode *mode = &tok->tok_mode_stack[index]; mode->start_offset = mode->start == NULL ? -1 : mode->start - tok->buf; mode->multi_line_start_offset = mode->multi_line_start == NULL @@ -23,19 +24,19 @@ _PyLexer_SnapshotBuffer(struct tok_state *tok, const char *base, } void -_PyLexer_RestoreBuffer(struct tok_state *tok, char *base, - const _PyLexer_BufferSnapshot *snapshot) +_PyLexer_RestoreBufferPointers(struct tok_state *tok, char *base, + const _PyLexer_BufferPointers *pointers) { - tok->buf = base + snapshot->buf; - tok->cur = tok->buf + snapshot->cur; - tok->inp = tok->buf + snapshot->inp; - tok->start = snapshot->start < 0 - ? NULL : tok->buf + snapshot->start; - tok->line_start = snapshot->line_start < 0 - ? NULL : tok->buf + snapshot->line_start; - tok->multi_line_start = snapshot->multi_line_start < 0 - ? NULL : tok->buf + snapshot->multi_line_start; - for (int index = tok->tok_mode_stack_index; index >= 0; --index) { + tok->buf = base + pointers->buf_from_base; + tok->cur = tok->buf + pointers->cur_from_buf; + tok->inp = tok->buf + pointers->inp_from_buf; + tok->start = pointers->start_from_buf < 0 + ? NULL : tok->buf + pointers->start_from_buf; + tok->line_start = pointers->line_start_from_buf < 0 + ? NULL : tok->buf + pointers->line_start_from_buf; + tok->multi_line_start = pointers->multi_line_start_from_buf < 0 + ? NULL : tok->buf + pointers->multi_line_start_from_buf; + for (int index = tok->tok_mode_stack_index; index > 0; --index) { tokenizer_mode *mode = &tok->tok_mode_stack[index]; mode->start = mode->start_offset < 0 ? NULL : tok->buf + mode->start_offset; diff --git a/Parser/lexer/buffer.h b/Parser/lexer/buffer.h index 06c54ff8da944a3..285da124226d50e 100644 --- a/Parser/lexer/buffer.h +++ b/Parser/lexer/buffer.h @@ -6,17 +6,17 @@ struct tok_state; typedef struct { - Py_ssize_t buf; - Py_ssize_t cur; - Py_ssize_t inp; - Py_ssize_t start; - Py_ssize_t line_start; - Py_ssize_t multi_line_start; -} _PyLexer_BufferSnapshot; + Py_ssize_t buf_from_base; + Py_ssize_t cur_from_buf; + Py_ssize_t inp_from_buf; + Py_ssize_t start_from_buf; + Py_ssize_t line_start_from_buf; + Py_ssize_t multi_line_start_from_buf; +} _PyLexer_BufferPointers; -void _PyLexer_SnapshotBuffer( - struct tok_state *, const char *, _PyLexer_BufferSnapshot *); -void _PyLexer_RestoreBuffer( - struct tok_state *, char *, const _PyLexer_BufferSnapshot *); +void _PyLexer_SaveBufferPointers( + struct tok_state *, const char *, _PyLexer_BufferPointers *); +void _PyLexer_RestoreBufferPointers( + struct tok_state *, char *, const _PyLexer_BufferPointers *); #endif diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c index 408bfda147b4e94..44dde9e4b0878e0 100644 --- a/Parser/tokenizer/reader.c +++ b/Parser/tokenizer/reader.c @@ -42,19 +42,6 @@ _PyTok_ReaderFree(struct tok_state *tok) tok->reader = NULL; } -static int -resize_buffer(char **buffer, Py_ssize_t *capacity, Py_ssize_t new_capacity) -{ - char *resized = PyMem_Realloc(*buffer, new_capacity); - if (resized == NULL) { - PyErr_NoMemory(); - return -1; - } - *buffer = resized; - *capacity = new_capacity; - return 0; -} - static int reserve_buffer(char **buffer, Py_ssize_t *capacity, Py_ssize_t needed) { @@ -69,7 +56,14 @@ reserve_buffer(char **buffer, Py_ssize_t *capacity, Py_ssize_t needed) } cap *= 2; } - return resize_buffer(buffer, capacity, cap); + char *resized = PyMem_Realloc(*buffer, cap); + if (resized == NULL) { + PyErr_NoMemory(); + return -1; + } + *buffer = resized; + *capacity = cap; + return 0; } static int @@ -82,18 +76,14 @@ reserve_input_buffer(struct tok_state *tok, Py_ssize_t needed) assert(tok->buf != NULL); assert(tok->cur >= tok->buf && tok->cur <= tok->inp); assert(tok->inp - tok->buf <= reader->input_buffer_cap); - Py_ssize_t used = tok->inp - tok->buf; - Py_ssize_t growth = used >> 1; - Py_ssize_t capacity = used > PY_SSIZE_T_MAX - growth - ? needed : Py_MAX(needed, used + growth); - _PyLexer_BufferSnapshot snapshot; - _PyLexer_SnapshotBuffer(tok, tok->buf, &snapshot); + _PyLexer_BufferPointers pointers; + _PyLexer_SaveBufferPointers(tok, tok->buf, &pointers); char *buffer = tok->buf; - if (resize_buffer( - &buffer, &reader->input_buffer_cap, capacity) < 0) { + if (reserve_buffer( + &buffer, &reader->input_buffer_cap, needed) < 0) { return -1; } - _PyLexer_RestoreBuffer(tok, buffer, &snapshot); + _PyLexer_RestoreBufferPointers(tok, buffer, &pointers); return 0; } @@ -642,10 +632,10 @@ _PyTok_ReaderUnderflow(struct tok_state *tok) else if (!prepared) { int source_will_grow = chunk.len > tok->source.cap - tok->source.len - 1; - _PyLexer_BufferSnapshot snapshot; + _PyLexer_BufferPointers pointers; if (!reset_buffer && source_will_grow) { - _PyLexer_SnapshotBuffer( - tok, tok->source.bytes, &snapshot); + _PyLexer_SaveBufferPointers( + tok, tok->source.bytes, &pointers); } _PyTok_Off source_start = _PyTok_SourceAppendLine( &tok->source, chunk.data, chunk.len, @@ -665,8 +655,8 @@ _PyTok_ReaderUnderflow(struct tok_state *tok) tok->multi_line_start = NULL; } else if (source_will_grow) { - _PyLexer_RestoreBuffer( - tok, tok->source.bytes, &snapshot); + _PyLexer_RestoreBufferPointers( + tok, tok->source.bytes, &pointers); } tok->inp = tok->source.bytes + source_start + scan_len; } diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index eee1fc86ded781f..7557daa49cbd56b 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -203,12 +203,16 @@ _get_current_line(tokenizeriterobject *it, const char *line_start, Py_ssize_t si } static void -_get_col_offsets(tokenizeriterobject *it, const char *token_start, - const char *token_end, const char *line_start, PyObject *line, - int line_changed, Py_ssize_t lineno, Py_ssize_t end_lineno, +_get_col_offsets(tokenizeriterobject *it, const struct token *token, + const char *token_start, const char *line_start, + PyObject *line, int line_changed, Py_ssize_t *col_offset, Py_ssize_t *end_col_offset) { _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(it); + const char *token_end = token_start == NULL + ? NULL : token_start + token->span.end - token->span.start; + Py_ssize_t lineno = token->start_loc.lineno; + Py_ssize_t end_lineno = token->end_loc.lineno; Py_ssize_t byte_offset = -1; if (token_start != NULL && token_start >= line_start) { byte_offset = token_start - line_start; @@ -263,7 +267,6 @@ tokenizeriter_next(PyObject *op) goto exit; } const char *token_start = NULL; - const char *token_end = NULL; PyObject *str; if (!_PyTok_SpanIsValid(token.span)) { str = Py_GetConstant(Py_CONSTANT_EMPTY_STR); @@ -272,7 +275,6 @@ tokenizeriter_next(PyObject *op) Py_ssize_t token_length; token_start = _PyToken_TextView( it->tok, &token, &token_length); - token_end = token_start + token_length; str = PyUnicode_FromStringAndSize(token_start, token_length); } if (str == NULL) { @@ -306,8 +308,8 @@ tokenizeriter_next(PyObject *op) Py_ssize_t end_lineno = token.end_loc.lineno; Py_ssize_t col_offset = -1; Py_ssize_t end_col_offset = -1; - _get_col_offsets(it, token_start, token_end, line_start, line, line_changed, - lineno, end_lineno, &col_offset, &end_col_offset); + _get_col_offsets(it, &token, token_start, line_start, line, line_changed, + &col_offset, &end_col_offset); if (it->tok->tok_extra_tokens) { if (is_trailing_token) { From fced7ab34921cf004bc6a353a3c3c30bcb746134 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sat, 29 Aug 2026 00:27:55 +0100 Subject: [PATCH 04/11] gh-153569: streamline tokenizer buffer hot paths --- Parser/lexer/lexer.h | 4 +++- Parser/tokenizer/reader.c | 5 ++--- Python/Python-tokenize.c | 3 ++- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Parser/lexer/lexer.h b/Parser/lexer/lexer.h index 232dbaa287470eb..040935a7e689138 100644 --- a/Parser/lexer/lexer.h +++ b/Parser/lexer/lexer.h @@ -14,10 +14,12 @@ _PyToken_TextView(const struct tok_state *tok, const struct token *token, Py_ssize_t *length) { assert(length != NULL); - if (!_PyTok_SpanIsValid(token->span)) { + if (token->span.start < 0) { + assert(token->span.start == -1 && token->span.end == -1); *length = 0; return ""; } + assert(_PyTok_SpanIsValid(token->span)); assert(tok->buf != NULL); assert(tok->inp >= tok->buf); assert(token->span.start >= tok->buf_offset); diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c index 44dde9e4b0878e0..68c8da2186ede96 100644 --- a/Parser/tokenizer/reader.c +++ b/Parser/tokenizer/reader.c @@ -78,12 +78,11 @@ reserve_input_buffer(struct tok_state *tok, Py_ssize_t needed) assert(tok->inp - tok->buf <= reader->input_buffer_cap); _PyLexer_BufferPointers pointers; _PyLexer_SaveBufferPointers(tok, tok->buf, &pointers); - char *buffer = tok->buf; if (reserve_buffer( - &buffer, &reader->input_buffer_cap, needed) < 0) { + &tok->buf, &reader->input_buffer_cap, needed) < 0) { return -1; } - _PyLexer_RestoreBufferPointers(tok, buffer, &pointers); + _PyLexer_RestoreBufferPointers(tok, tok->buf, &pointers); return 0; } diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index 7557daa49cbd56b..71f236b08d93c8f 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -268,7 +268,8 @@ tokenizeriter_next(PyObject *op) } const char *token_start = NULL; PyObject *str; - if (!_PyTok_SpanIsValid(token.span)) { + if (token.span.start < 0) { + assert(token.span.start == -1 && token.span.end == -1); str = Py_GetConstant(Py_CONSTANT_EMPTY_STR); } else { From 64893b46a65b4bee3623ef8190423bb067975799 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Fri, 28 Aug 2026 19:27:20 +0100 Subject: [PATCH 05/11] gh-153569: remove unused tokenizer cursor support --- Lib/test/test_capi/test_tokenizer.py | 3 - Makefile.pre.in | 4 +- Modules/_testinternalcapi/tokenizer.c | 252 ++------------------------ PCbuild/pythoncore.vcxproj | 2 - PCbuild/pythoncore.vcxproj.filters | 6 - Parser/tokenizer/cursor.c | 81 --------- Parser/tokenizer/cursor.h | 65 ------- Parser/tokenizer/source.c | 158 +--------------- Parser/tokenizer/source.h | 43 +---- 9 files changed, 17 insertions(+), 597 deletions(-) delete mode 100644 Parser/tokenizer/cursor.c delete mode 100644 Parser/tokenizer/cursor.h diff --git a/Lib/test/test_capi/test_tokenizer.py b/Lib/test/test_capi/test_tokenizer.py index 2fe1fef241e90ae..e986e0f6b74f407 100644 --- a/Lib/test/test_capi/test_tokenizer.py +++ b/Lib/test/test_capi/test_tokenizer.py @@ -9,9 +9,6 @@ class TokenizerTests(unittest.TestCase): def test_source(self): _testinternalcapi.test_tokenizer_source() - def test_cursor(self): - _testinternalcapi.test_tokenizer_cursor() - if __name__ == "__main__": unittest.main() diff --git a/Makefile.pre.in b/Makefile.pre.in index adcfe4c5259eb22..982b2d66216c982 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -399,7 +399,6 @@ TOKENIZER_OBJS= \ Parser/lexer/number.o \ Parser/lexer/state.o \ Parser/lexer/string.o \ - Parser/tokenizer/cursor.o \ Parser/tokenizer/decoder.o \ Parser/tokenizer/reader.o \ Parser/tokenizer/source.o \ @@ -415,7 +414,6 @@ TOKENIZER_HEADERS= \ Parser/lexer/lexer.h \ Parser/lexer/lexer_internal.h \ Parser/lexer/state.h \ - Parser/tokenizer/cursor.h \ Parser/tokenizer/reader.h \ Parser/tokenizer/reader_internal.h \ Parser/tokenizer/source.h \ @@ -3462,7 +3460,7 @@ MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo. MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/parts.h $(srcdir)/Modules/_testcapi/util.h MODULE__TESTLIMITEDCAPI_DEPS=$(srcdir)/Modules/_testlimitedcapi/testcapi_long.h $(srcdir)/Modules/_testlimitedcapi/parts.h $(srcdir)/Modules/_testlimitedcapi/util.h -MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h $(srcdir)/Parser/tokenizer/cursor.h $(srcdir)/Parser/tokenizer/source.h $(srcdir)/Python/ceval.h $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h +MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h $(srcdir)/Parser/tokenizer/source.h $(srcdir)/Python/ceval.h $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h MODULE__SQLITE3_DEPS=$(srcdir)/Modules/_sqlite/connection.h $(srcdir)/Modules/_sqlite/cursor.h $(srcdir)/Modules/_sqlite/microprotocols.h $(srcdir)/Modules/_sqlite/module.h $(srcdir)/Modules/_sqlite/prepare_protocol.h $(srcdir)/Modules/_sqlite/row.h $(srcdir)/Modules/_sqlite/util.h MODULE__ZSTD_DEPS=$(srcdir)/Modules/_zstd/_zstdmodule.h $(srcdir)/Modules/_zstd/buffer.h $(srcdir)/Modules/_zstd/zstddict.h diff --git a/Modules/_testinternalcapi/tokenizer.c b/Modules/_testinternalcapi/tokenizer.c index 0b292410d3eb4ef..46ac3c40724cff1 100644 --- a/Modules/_testinternalcapi/tokenizer.c +++ b/Modules/_testinternalcapi/tokenizer.c @@ -1,6 +1,6 @@ #include "parts.h" -#include "../../Parser/tokenizer/cursor.h" +#include "../../Parser/tokenizer/source.h" static int check(int condition, const char *message) @@ -23,16 +23,6 @@ check_system_error(int failed, const char *message) return 0; } -static int -same_cursor(const _PyTok_Cursor *left, const _PyTok_Cursor *right) -{ - return left->source == right->source && - left->pos == right->pos && - left->line_start == right->line_start && - left->line_end == right->line_end && - left->lineno == right->lineno; -} - static PyObject * test_tokenizer_source(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) @@ -40,80 +30,24 @@ test_tokenizer_source(PyObject *Py_UNUSED(module), _PyTok_SourceText source; _PyTok_SourceInit(&source); - _PyTok_Loc loc; - _PyTok_Line line; - if (check(_PyTok_SourceLocation( - &source, 0, _PYTOK_AFFINITY_RIGHT, &loc) == 0, - "cannot locate empty source") < 0 || - check(loc.lineno == 1 && loc.byte_col == 0, - "wrong empty source location") < 0 || - check(_PyTok_SourceLine(&source, 1, &line) == 0, - "cannot find empty source line") < 0 || - check(line.start == 0 && line.end == 0, - "wrong empty source line") < 0 || - check_system_error( - _PyTok_SourceAppendLine(&source, "", 0, 0) < 0, - "accepted empty source line") < 0 || + if (check_system_error( + _PyTok_SourceAppendLine(&source, "", 0, 0) < 0, + "accepted empty source line") < 0 || check_system_error( _PyTok_SourceAppendLine(&source, "a\nb\n", 4, 0) < 0, "accepted multiple source lines") < 0 || check_system_error( _PyTok_SourceAppendLine(&source, "a", 1, 1) < 0, - "accepted missing implicit newline") < 0) { - goto error; - } - - if (check(_PyTok_SourceAppendLine(&source, "alpha\n", 6, 0) == 0, - "wrong first source offset") < 0 || + "accepted missing implicit newline") < 0 || + check(_PyTok_SourceAppendLine( + &source, "alpha\n", 6, 0) == 0, + "wrong first source offset") < 0 || check(_PyTok_SourceAppendLine( &source, "\xce\xb2\n", 3, 1) == 6, "wrong second source offset") < 0 || - check(_PyTok_SourceAppendLine( - &source, "nul\0x\n", 6, 0) == 9, - "wrong third source offset") < 0) { - goto error; - } - - int marker_line = 257; - int final_line = 300; - _PyTok_Off marker_start = -1; - for (int lineno = 4; lineno <= final_line; lineno++) { - const char *text = lineno == marker_line ? "marker\n" : "x\n"; - Py_ssize_t len = (Py_ssize_t)strlen(text); - _PyTok_Off start = _PyTok_SourceAppendLine( - &source, text, len, lineno == final_line); - if (start < 0) { - goto error; - } - if (lineno == marker_line) { - marker_start = start; - } - } - - if (check(source.nlines == final_line, "wrong source line count") < 0 || - check(_PyTok_SourceLine(&source, marker_line, &line) == 0, - "cannot find late source line") < 0 || - check(line.start == marker_start && - line.end == marker_start + 7, - "wrong late source line") < 0 || - check(!line.implicit_newline && !line.contains_nul, - "wrong late source flags") < 0 || - check(_PyTok_SourceLine(&source, 2, &line) == 0, - "cannot find second source line") < 0 || - check(line.start == 6 && line.end == 9 && - line.implicit_newline && !line.contains_nul, - "wrong second source line") < 0 || check(!_PyTok_SourceLineIsImplicit(&source, 1) && _PyTok_SourceLineIsImplicit(&source, 2), - "wrong early implicit newline flags") < 0 || - check(_PyTok_SourceLine(&source, 3, &line) == 0, - "cannot find third source line") < 0 || - check(line.contains_nul, "missing null byte flag") < 0 || - check(_PyTok_SourceLine(&source, final_line, &line) == 0, - "cannot find final source line") < 0 || - check(line.implicit_newline && - _PyTok_SourceLineIsImplicit(&source, final_line), - "missing late implicit newline flag") < 0) { + "wrong implicit newline flags") < 0) { goto error; } @@ -123,178 +57,21 @@ test_tokenizer_source(PyObject *Py_UNUSED(module), if (check(view != NULL && view_len == 2 && memcmp(view, "\xce\xb2", 2) == 0, "wrong source span view") < 0 || - check(_PyTok_SourceLocation( - &source, marker_start, - _PYTOK_AFFINITY_LEFT, &loc) == 0, - "cannot locate left line boundary") < 0 || - check(loc.lineno == marker_line - 1 && loc.byte_col == 2, - "wrong left boundary location") < 0 || - check(_PyTok_SourceLocation( - &source, marker_start, - _PYTOK_AFFINITY_RIGHT, &loc) == 0, - "cannot locate right line boundary") < 0 || - check(loc.lineno == marker_line && loc.byte_col == 0, - "wrong right boundary location") < 0 || - check(_PyTok_SourceLocation( - &source, marker_start + 1, - _PYTOK_AFFINITY_RIGHT, &loc) == 0, - "cannot locate late source byte") < 0 || - check(loc.lineno == marker_line && loc.byte_col == 1, - "wrong late source location") < 0) { - goto error; - } - - if (check(_PyTok_SourceLocation( - &source, source.len, _PYTOK_AFFINITY_LEFT, &loc) == 0, - "cannot locate left EOF") < 0 || - check(loc.lineno == final_line && loc.byte_col == 2, - "wrong left EOF location") < 0 || - check(_PyTok_SourceLocation( - &source, source.len, - _PYTOK_AFFINITY_RIGHT, &loc) == 0, - "cannot locate right EOF") < 0 || - check(loc.lineno == final_line + 1 && loc.byte_col == 0, - "wrong right EOF location") < 0 || - check(_PyTok_SourceLine(&source, final_line + 1, &line) == 0, - "cannot find virtual EOF line") < 0 || - check(line.start == source.len && line.end == source.len, - "wrong virtual EOF line") < 0 || - check(!_PyTok_SourceLineIsImplicit(&source, 0) && - !_PyTok_SourceLineIsImplicit( - &source, final_line + 1), - "virtual or invalid line is implicit") < 0) { - goto error; - } - - view = _PyTok_SourceSpanView( - &source, _PyTok_SpanFromBounds(0, source.len + 1), &view_len); - if (check_system_error(view == NULL, "accepted invalid source span") < 0 || check_system_error( - _PyTok_SourceLocation( - &source, source.len + 1, - _PYTOK_AFFINITY_RIGHT, &loc) < 0, - "accepted invalid source offset") < 0 || - check_system_error( - _PyTok_SourceLine(&source, final_line + 2, &line) < 0, - "accepted invalid source line") < 0) { + _PyTok_SourceSpanView( + &source, _PyTok_SpanFromBounds(0, source.len + 1), + &view_len) == NULL, + "accepted invalid source span") < 0) { goto error; } _PyTok_SourceClear(&source); - _PyTok_SourceInit(&source); if (_PyTok_SourceAppendLine(&source, "tail", 4, 0) < 0 || check_system_error( _PyTok_SourceAppendLine(&source, "x\n", 2, 0) < 0, - "appended after unterminated source line") < 0 || - check(_PyTok_SourceLocation( - &source, source.len, - _PYTOK_AFFINITY_RIGHT, &loc) == 0, - "cannot locate unterminated EOF") < 0 || - check(loc.lineno == 1 && loc.byte_col == 4, - "wrong unterminated EOF location") < 0) { - goto error; - } - - _PyTok_SourceClear(&source); - Py_RETURN_NONE; - -error: - _PyTok_SourceClear(&source); - return NULL; -} - -static PyObject * -test_tokenizer_cursor(PyObject *Py_UNUSED(module), - PyObject *Py_UNUSED(args)) -{ - _PyTok_SourceText source; - _PyTok_SourceInit(&source); - if (_PyTok_SourceAppendLine(&source, "ab\n", 3, 0) < 0 || - _PyTok_SourceAppendLine(&source, "cd\n", 3, 0) < 0) { - goto error; - } - - _PyTok_Cursor cursor; - _PyTok_CursorInit(&cursor, &source); - if (_PyTok_CursorSetOffset(&cursor, source.len) < 0 || - check(cursor.lineno == 3 && cursor.pos == source.len, - "wrong cursor at virtual EOF") < 0 || - _PyTok_CursorSetLine(&cursor, 1) < 0) { - goto error; - } - - char large[BUFSIZ + 1]; - memset(large, 'z', sizeof(large)); - large[sizeof(large) - 1] = '\n'; - if (_PyTok_SourceAppendLine(&source, large, sizeof(large), 0) < 0) { - goto error; - } - - if (check(_PyTok_CursorPeek(&cursor, 0) == 'a', - "wrong cursor peek after relocation") < 0 || - check(_PyTok_CursorPeek(&cursor, 1) == 'b', - "wrong distant cursor peek") < 0 || - check(_PyTok_CursorAdvance(&cursor) == 'a', - "wrong first cursor byte") < 0 || - check(_PyTok_CursorAdvance(&cursor) == 'b', - "wrong second cursor byte") < 0 || - check(_PyTok_CursorAdvance(&cursor) == '\n', - "wrong final cursor byte") < 0 || - check(_PyTok_CursorAdvance(&cursor) == EOF, - "cursor advanced past line") < 0 || - check(_PyTok_CursorSetOffset(&cursor, 2) == 0, - "cannot seek cursor offset") < 0 || - check(_PyTok_CursorAdvance(&cursor) == '\n', - "wrong cursor byte after seek") < 0 || - check(_PyTok_CursorSetOffset(&cursor, 3) == 0, - "cannot seek line boundary") < 0 || - check(cursor.lineno == 2 && cursor.line_start == 3 && - _PyTok_CursorAdvance(&cursor) == 'c', - "wrong cursor at line boundary") < 0 || - check(_PyTok_CursorSetLine(&cursor, 3) == 0, - "cannot advance cursor to final line") < 0 || - check(cursor.line_start == 6 && - _PyTok_CursorAdvance(&cursor) == 'z', - "wrong cursor byte on final line") < 0) { - goto error; - } - - _PyTok_Cursor saved = cursor; - if (check_system_error( - _PyTok_CursorSetOffset(&cursor, source.len + 1) < 0, - "accepted invalid cursor offset") < 0 || - check(same_cursor(&cursor, &saved), - "invalid offset changed cursor") < 0 || - check_system_error( - _PyTok_CursorSetLine(&cursor, source.nlines + 2) < 0, - "accepted invalid cursor line") < 0 || - check(same_cursor(&cursor, &saved), - "invalid line changed cursor") < 0 || - check(_PyTok_CursorSetOffset(&cursor, source.len) == 0, - "cannot set cursor to EOF") < 0 || - check(cursor.lineno == 4 && cursor.pos == source.len, - "wrong cursor at EOF") < 0) { - goto error; - } - -#if SIZEOF_VOID_P > 4 - char byte = 0; - _PyTok_SourceText huge_source = { - .bytes = &byte, - .len = (_PyTok_Off)INT_MAX + 1, - }; - _PyTok_Cursor huge_cursor = { - .source = &huge_source, - .pos = INT_MAX, - .line_end = (_PyTok_Off)INT_MAX + 1, - .lineno = 1, - }; - if (check(_PyTok_CursorAdvance(&huge_cursor) == EOF && - huge_cursor.pos == INT_MAX, - "cursor advanced past maximum column") < 0) { + "appended after unterminated source line") < 0) { goto error; } -#endif _PyTok_SourceClear(&source); Py_RETURN_NONE; @@ -306,7 +83,6 @@ test_tokenizer_cursor(PyObject *Py_UNUSED(module), static PyMethodDef test_methods[] = { {"test_tokenizer_source", test_tokenizer_source, METH_NOARGS}, - {"test_tokenizer_cursor", test_tokenizer_cursor, METH_NOARGS}, {NULL}, }; diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 33647ec284061f1..93dd56a8ef166f6 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -423,7 +423,6 @@ - @@ -593,7 +592,6 @@ - diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 434dd13267fe934..f97b51a0e48a3eb 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -333,9 +333,6 @@ Parser - - Parser - Parser @@ -1361,9 +1358,6 @@ Parser - - Parser - Parser diff --git a/Parser/tokenizer/cursor.c b/Parser/tokenizer/cursor.c deleted file mode 100644 index 698a26a740fd249..000000000000000 --- a/Parser/tokenizer/cursor.c +++ /dev/null @@ -1,81 +0,0 @@ -#include "Python.h" - -#include "cursor.h" - -static void -set_line(_PyTok_Cursor *cursor, int lineno, _PyTok_Off start, - _PyTok_Off end) -{ - cursor->pos = start; - cursor->line_start = start; - cursor->line_end = end; - cursor->lineno = lineno; -} - -int -_PyTok_CursorSetLine(_PyTok_Cursor *cursor, int lineno) -{ - if (cursor->source == NULL) { - PyErr_SetString(PyExc_SystemError, "cursor has no tokenizer source"); - return -1; - } - const _PyTok_SourceText *source = cursor->source; - if (lineno > 0 && cursor->lineno == lineno - 1 && - lineno <= source->nlines) { - _PyTok_Off start = cursor->line_end; - _PyTok_Off end = source->len; - if (lineno < source->nlines) { - end = _PyTok_SourceFindLineEnd(source, start); - if (end < 0) { - return -1; - } - } - set_line(cursor, lineno, start, end); - return 0; - } - - _PyTok_Line line; - if (_PyTok_SourceLine(source, lineno, &line) < 0) { - return -1; - } - set_line(cursor, lineno, line.start, line.end); - return 0; -} - -int -_PyTok_CursorSetOffset(_PyTok_Cursor *cursor, _PyTok_Off offset) -{ - if (cursor->source == NULL) { - PyErr_SetString(PyExc_SystemError, "cursor has no tokenizer source"); - return -1; - } - const _PyTok_SourceText *source = cursor->source; - int stays_on_line = cursor->lineno > 0 && - offset >= cursor->line_start && offset < cursor->line_end; - if (!stays_on_line && cursor->lineno > 0 && - offset == cursor->line_end && offset == source->len && - (offset == 0 || source->bytes[offset - 1] != '\n')) { - stays_on_line = 1; - } - if (stays_on_line) { - cursor->pos = offset; - return 0; - } - - _PyTok_Loc loc; - if (_PyTok_SourceLocation( - source, offset, _PYTOK_AFFINITY_RIGHT, &loc) < 0) { - return -1; - } - _PyTok_Off start = offset - loc.byte_col; - _PyTok_Off end = source->len; - if (loc.lineno < source->nlines) { - end = _PyTok_SourceFindLineEnd(source, start); - if (end < 0) { - return -1; - } - } - set_line(cursor, loc.lineno, start, end); - cursor->pos = offset; - return 0; -} diff --git a/Parser/tokenizer/cursor.h b/Parser/tokenizer/cursor.h deleted file mode 100644 index d0fd9cf80b77b71..000000000000000 --- a/Parser/tokenizer/cursor.h +++ /dev/null @@ -1,65 +0,0 @@ -#ifndef Py_TOKENIZER_CURSOR_H -#define Py_TOKENIZER_CURSOR_H - -#include "source.h" - -typedef struct { - /* The source must remain initialized at this address while in use. */ - const _PyTok_SourceText *source; - _PyTok_Off pos; - _PyTok_Off line_start; - _PyTok_Off line_end; - int lineno; -} _PyTok_Cursor; - -/* Move to the start of a 1-based line. Both setters preserve the cursor on - error. */ -PyAPI_FUNC(int) _PyTok_CursorSetLine(_PyTok_Cursor *, int); -/* Move to an offset. A line boundary selects the following line. */ -PyAPI_FUNC(int) _PyTok_CursorSetOffset(_PyTok_Cursor *, _PyTok_Off); - -static inline void -_PyTok_CursorInit(_PyTok_Cursor *cursor, const _PyTok_SourceText *source) -{ - *cursor = (_PyTok_Cursor){ - .source = source, - }; -} - -/* Read one byte from the current line, including its terminating newline. - EOF marks the line boundary, not necessarily the end of the source. It is - also returned if advancing would make the byte column unrepresentable. */ -static inline int -_PyTok_CursorAdvance(_PyTok_Cursor *cursor) -{ - assert(cursor->source != NULL); - assert(cursor->pos >= cursor->line_start); - assert(cursor->pos <= cursor->line_end); - assert(cursor->line_end <= cursor->source->len); - if (cursor->pos >= cursor->line_end) { - return EOF; - } - if (cursor->pos - cursor->line_start >= INT_MAX) { - return EOF; - } - return Py_CHARMASK(cursor->source->bytes[cursor->pos++]); -} - -/* Return the byte at a nonnegative distance within the current line, or EOF - if the distance reaches or crosses the line boundary. */ -static inline int -_PyTok_CursorPeek(const _PyTok_Cursor *cursor, int distance) -{ - assert(cursor->source != NULL); - assert(cursor->pos >= cursor->line_start); - assert(cursor->pos <= cursor->line_end); - assert(cursor->line_end <= cursor->source->len); - assert(distance >= 0); - if (distance < 0 || - distance >= cursor->line_end - cursor->pos) { - return EOF; - } - return Py_CHARMASK(cursor->source->bytes[cursor->pos + distance]); -} - -#endif diff --git a/Parser/tokenizer/source.c b/Parser/tokenizer/source.c index c0f7925e33f8b97..e876be6e0026519 100644 --- a/Parser/tokenizer/source.c +++ b/Parser/tokenizer/source.c @@ -2,8 +2,6 @@ #include "source.h" -#define LINE_CHECKPOINT_INTERVAL 256 - void _PyTok_SourceInit(_PyTok_SourceText *source) { @@ -14,7 +12,6 @@ void _PyTok_SourceClear(_PyTok_SourceText *source) { PyMem_Free(source->bytes); - PyMem_Free(source->line_checkpoints); PyMem_Free(source->implicit_lines); _PyTok_SourceInit(source); } @@ -58,34 +55,6 @@ reserve_bytes(_PyTok_SourceText *source, Py_ssize_t needed) return 0; } -static int -reserve_checkpoints(_PyTok_SourceText *source, int needed) -{ - if (needed <= source->checkpoints_cap) { - return 0; - } - int cap; - if (source->checkpoints_cap == 0) { - cap = 16; - } - else if (source->checkpoints_cap <= INT_MAX / 2) { - cap = source->checkpoints_cap * 2; - } - else { - PyErr_NoMemory(); - return -1; - } - _PyTok_Off *checkpoints = source->line_checkpoints; - PyMem_Resize(checkpoints, _PyTok_Off, cap); - if (checkpoints == NULL) { - PyErr_NoMemory(); - return -1; - } - source->line_checkpoints = checkpoints; - source->checkpoints_cap = cap; - return 0; -} - static int reserve_implicit_lines(_PyTok_SourceText *source, int nlines) { @@ -148,11 +117,7 @@ _PyTok_SourceAppendLine(_PyTok_SourceText *source, const char *bytes, return -1; } int nlines = source->nlines + 1; - int checkpoint = ((nlines - 1) % LINE_CHECKPOINT_INTERVAL) == 0; - int checkpoint_count = (nlines - 1) / LINE_CHECKPOINT_INTERVAL + 1; - if ((checkpoint && - reserve_checkpoints(source, checkpoint_count) < 0) || - (implicit_newline && reserve_implicit_lines(source, nlines) < 0) || + if ((implicit_newline && reserve_implicit_lines(source, nlines) < 0) || reserve_bytes(source, source->len + len + 1) < 0) { return -1; } @@ -161,9 +126,6 @@ _PyTok_SourceAppendLine(_PyTok_SourceText *source, const char *bytes, memcpy(source->bytes + start, bytes, len); source->len += len; source->bytes[source->len] = '\0'; - if (checkpoint) { - source->line_checkpoints[checkpoint_count - 1] = start; - } if (implicit_newline) { source->implicit_lines[(nlines - 1) / 8] |= (unsigned char)(1U << ((nlines - 1) & 7)); @@ -194,121 +156,3 @@ _PyTok_SourceLineIsImplicit(const _PyTok_SourceText *source, int lineno) return (source->implicit_lines[(lineno - 1) / 8] >> ((lineno - 1) & 7)) & 1; } - -static int -source_ends_in_newline(const _PyTok_SourceText *source) -{ - return source->len > 0 && source->bytes[source->len - 1] == '\n'; -} - -static int -eof_lineno(const _PyTok_SourceText *source) -{ - if (source->nlines == 0) { - return 1; - } - return source->nlines + source_ends_in_newline(source); -} - -int -_PyTok_SourceLine(const _PyTok_SourceText *source, int lineno, - _PyTok_Line *line) -{ - if (line == NULL || lineno < 1 || lineno > eof_lineno(source)) { - PyErr_SetString(PyExc_SystemError, "invalid tokenizer source line"); - return -1; - } - if (lineno > source->nlines) { - *line = (_PyTok_Line){ - .start = source->len, - .end = source->len, - }; - return 0; - } - - int checkpoint = (lineno - 1) / LINE_CHECKPOINT_INTERVAL; - int current = checkpoint * LINE_CHECKPOINT_INTERVAL + 1; - _PyTok_Off start = source->line_checkpoints[checkpoint]; - while (current < lineno) { - start = _PyTok_SourceFindLineEnd(source, start); - if (start < 0) { - return -1; - } - current++; - } - _PyTok_Off end = source->len; - if (lineno < source->nlines) { - end = _PyTok_SourceFindLineEnd(source, start); - if (end < 0) { - return -1; - } - } - *line = (_PyTok_Line){ - .start = start, - .end = end, - .implicit_newline = _PyTok_SourceLineIsImplicit(source, lineno), - .contains_nul = memchr( - source->bytes + start, 0, end - start) != NULL, - }; - return 0; -} - -int -_PyTok_SourceLocation(const _PyTok_SourceText *source, _PyTok_Off offset, - _PyTok_Affinity affinity, _PyTok_Loc *loc) -{ - if (offset < 0 || offset > source->len || loc == NULL || - (affinity != _PYTOK_AFFINITY_LEFT && - affinity != _PYTOK_AFFINITY_RIGHT)) { - PyErr_SetString(PyExc_SystemError, "invalid tokenizer source offset"); - return -1; - } - if (source->nlines == 0 || - (offset == source->len && source_ends_in_newline(source) && - affinity == _PYTOK_AFFINITY_RIGHT)) { - *loc = (_PyTok_Loc){eof_lineno(source), 0}; - return 0; - } - - _PyTok_Off key = offset; - if (affinity == _PYTOK_AFFINITY_LEFT && key > 0) { - key--; - } - int low = 0; - int high = (source->nlines - 1) / LINE_CHECKPOINT_INTERVAL + 1; - while (low < high) { - int middle = low + (high - low) / 2; - if (source->line_checkpoints[middle] <= key) { - low = middle + 1; - } - else { - high = middle; - } - } - int checkpoint = low - 1; - if (checkpoint < 0) { - PyErr_SetString(PyExc_SystemError, "corrupt tokenizer source line index"); - return -1; - } - int lineno = checkpoint * LINE_CHECKPOINT_INTERVAL + 1; - _PyTok_Off start = source->line_checkpoints[checkpoint]; - while (lineno < source->nlines) { - _PyTok_Off end = _PyTok_SourceFindLineEnd(source, start); - if (end < 0) { - return -1; - } - if (offset < end || - (offset == end && affinity == _PYTOK_AFFINITY_LEFT)) { - break; - } - start = end; - lineno++; - } - _PyTok_Off byte_col = offset - start; - if (byte_col > INT_MAX) { - PyErr_SetString(PyExc_OverflowError, "tokenizer column is too large"); - return -1; - } - *loc = (_PyTok_Loc){lineno, (int)byte_col}; - return 0; -} diff --git a/Parser/tokenizer/source.h b/Parser/tokenizer/source.h index 363475ff9015e35..fac17183ceb734e 100644 --- a/Parser/tokenizer/source.h +++ b/Parser/tokenizer/source.h @@ -18,32 +18,17 @@ typedef struct { int byte_col; } _PyTok_Loc; -typedef enum { - _PYTOK_AFFINITY_LEFT, - _PYTOK_AFFINITY_RIGHT, -} _PyTok_Affinity; - -/* The half-open range includes the terminating newline when present. */ -typedef struct { - _PyTok_Off start; - _PyTok_Off end; - unsigned implicit_newline : 1; - unsigned contains_nul : 1; -} _PyTok_Line; - typedef struct { char *bytes; _PyTok_Off len; _PyTok_Off cap; - _PyTok_Off *line_checkpoints; unsigned char *implicit_lines; int nlines; - int checkpoints_cap; Py_ssize_t implicit_cap; } _PyTok_SourceText; PyAPI_FUNC(void) _PyTok_SourceInit(_PyTok_SourceText *); -/* Clear invalidates all cursors, spans, and views for the source. */ +/* Clear invalidates all spans and views for the source. */ PyAPI_FUNC(void) _PyTok_SourceClear(_PyTok_SourceText *); /* Append one nonempty logical line and return its start offset. The input may contain one newline, as its final byte. An unterminated line must be the @@ -55,17 +40,9 @@ PyAPI_FUNC(_PyTok_Off) _PyTok_SourceAppendLine( /* The returned view is invalidated by SourceAppendLine and SourceClear. */ PyAPI_FUNC(const char *) _PyTok_SourceSpanView( const _PyTok_SourceText *, _PyTok_Span, Py_ssize_t *); -/* Look up a 1-based line. Empty and newline-terminated sources have an empty - virtual line at EOF. */ -PyAPI_FUNC(int) _PyTok_SourceLine( - const _PyTok_SourceText *, int, _PyTok_Line *); /* Return false for invalid line numbers and the virtual EOF line. */ PyAPI_FUNC(int) _PyTok_SourceLineIsImplicit( const _PyTok_SourceText *, int); -/* At a line boundary, left affinity selects the preceding line at its end; - right affinity selects the following line at byte column zero. */ -PyAPI_FUNC(int) _PyTok_SourceLocation( - const _PyTok_SourceText *, _PyTok_Off, _PyTok_Affinity, _PyTok_Loc *); static inline _PyTok_Span _PyTok_SpanFromBounds(_PyTok_Off start, _PyTok_Off end) @@ -79,22 +56,4 @@ _PyTok_SpanIsValid(_PyTok_Span span) return span.start >= 0 && span.end >= span.start; } -static inline _PyTok_Off -_PyTok_SourceFindLineEnd(const _PyTok_SourceText *source, _PyTok_Off start) -{ - if (source->bytes == NULL || start < 0 || start >= source->len) { - PyErr_SetString(PyExc_SystemError, - "corrupt tokenizer source line index"); - return -1; - } - const char *newline = memchr( - source->bytes + start, '\n', source->len - start); - if (newline == NULL) { - PyErr_SetString(PyExc_SystemError, - "corrupt tokenizer source line index"); - return -1; - } - return newline - source->bytes + 1; -} - #endif From d3290a7e7176d39ac18006fb183f2b5a8f2e933d Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Fri, 28 Aug 2026 19:27:25 +0100 Subject: [PATCH 06/11] gh-153569: store f-string state as source spans --- Lib/test/test_fstring.py | 6 + Lib/test/test_tstring.py | 8 ++ Parser/lexer/buffer.c | 13 -- Parser/lexer/lexer.c | 4 +- Parser/lexer/lexer.h | 26 ++-- Parser/lexer/lexer_internal.h | 1 + Parser/lexer/state.c | 19 --- Parser/lexer/state.h | 11 +- Parser/lexer/string.c | 248 +++++++++++++--------------------- Parser/tokenizer/reader.c | 6 - 10 files changed, 134 insertions(+), 208 deletions(-) diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index c1ef1a73f05c204..debd9a41063e49d 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -1679,6 +1679,12 @@ def __repr__(self): self.assertEqual(f'{" # nooo "=}', '" # nooo "=\' # nooo \'') self.assertEqual(f'{" \" # nooo \" "=}', '" \\" # nooo \\" "=\' " # nooo " \'') + self.assertEqual(f'{"""a" # inside"""=}', + '"""a" # inside"""=\'a" # inside\'') + self.assertEqual(f"{'''a' # inside'''=}", + "'''a' # inside'''=\"a' # inside\"") + self.assertEqual(f'{"""a""""#" # outside +=}', '"""a""""#" \n=\'a#\'') self.assertEqual(f'{ # some comment goes here """hello"""=}', ' \n """hello"""=\'hello\'') diff --git a/Lib/test/test_tstring.py b/Lib/test/test_tstring.py index 74653c77c55de17..b60b50e446e3656 100644 --- a/Lib/test/test_tstring.py +++ b/Lib/test/test_tstring.py @@ -287,5 +287,13 @@ def test_triple_quoted(self): ) self.assertEqual(fstring(t), "\n Hello,\n Python\n ") + t = t'{"""a" # inside"""}' + self.assertEqual(t.interpolations[0].expression, + '"""a" # inside"""') + + t = t'{"""a""""#" # outside +}' + self.assertEqual(t.interpolations[0].expression, '"""a""""#"') + if __name__ == '__main__': unittest.main() diff --git a/Parser/lexer/buffer.c b/Parser/lexer/buffer.c index 9c39544ca7c4790..7e2330482ec85dc 100644 --- a/Parser/lexer/buffer.c +++ b/Parser/lexer/buffer.c @@ -15,12 +15,6 @@ _PyLexer_SaveBufferPointers(struct tok_state *tok, const char *base, ? -1 : tok->line_start - tok->buf; pointers->multi_line_start_from_buf = tok->multi_line_start == NULL ? -1 : tok->multi_line_start - tok->buf; - for (int index = tok->tok_mode_stack_index; index > 0; --index) { - tokenizer_mode *mode = &tok->tok_mode_stack[index]; - mode->start_offset = mode->start == NULL ? -1 : mode->start - tok->buf; - mode->multi_line_start_offset = mode->multi_line_start == NULL - ? -1 : mode->multi_line_start - tok->buf; - } } void @@ -36,11 +30,4 @@ _PyLexer_RestoreBufferPointers(struct tok_state *tok, char *base, ? NULL : tok->buf + pointers->line_start_from_buf; tok->multi_line_start = pointers->multi_line_start_from_buf < 0 ? NULL : tok->buf + pointers->multi_line_start_from_buf; - for (int index = tok->tok_mode_stack_index; index > 0; --index) { - tokenizer_mode *mode = &tok->tok_mode_stack[index]; - mode->start = mode->start_offset < 0 - ? NULL : tok->buf + mode->start_offset; - mode->multi_line_start = mode->multi_line_start_offset < 0 - ? NULL : tok->buf + mode->multi_line_start_offset; - } } diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index 16f0a7b783f4b48..79b4de007c6daf1 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -555,8 +555,8 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str int cursor_in_format_with_debug = cursor == 1 && (current_tok->in_debug || in_format_spec); int cursor_valid = cursor == 0 || cursor_in_format_with_debug; - if ((cursor_valid) && !_PyLexer_update_ftstring_expr(tok, c)) { - return MAKE_TOKEN(ENDMARKER); + if (cursor_valid) { + _PyLexer_update_ftstring_expr(tok, c); } if ((cursor_valid) && c != '{' && _PyLexer_set_ftstring_expr(tok, token, c)) { return MAKE_TOKEN(ERRORTOKEN); diff --git a/Parser/lexer/lexer.h b/Parser/lexer/lexer.h index 040935a7e689138..776504c7c186777 100644 --- a/Parser/lexer/lexer.h +++ b/Parser/lexer/lexer.h @@ -3,10 +3,24 @@ #include "state.h" -int _PyLexer_update_ftstring_expr(struct tok_state *tok, char cur); - int _PyTokenizer_Get(struct tok_state *, struct token *); +static inline const char * +_PyLexer_SpanView(const struct tok_state *tok, _PyTok_Span span, + Py_ssize_t *length) +{ + assert(length != NULL); + assert(_PyTok_SpanIsValid(span)); + assert(tok->buf != NULL); + assert(tok->inp >= tok->buf); + if (span.start >= tok->buf_offset && + span.end - tok->buf_offset <= tok->inp - tok->buf) { + *length = span.end - span.start; + return tok->buf + (span.start - tok->buf_offset); + } + return _PyTok_SourceSpanView(&tok->source, span, length); +} + /* The view points into the current input window. The next _PyTokenizer_Get() call may discard it. */ static inline const char * @@ -19,13 +33,7 @@ _PyToken_TextView(const struct tok_state *tok, const struct token *token, *length = 0; return ""; } - assert(_PyTok_SpanIsValid(token->span)); - assert(tok->buf != NULL); - assert(tok->inp >= tok->buf); - assert(token->span.start >= tok->buf_offset); - assert(token->span.end - tok->buf_offset <= tok->inp - tok->buf); - *length = token->span.end - token->span.start; - return tok->buf + (token->span.start - tok->buf_offset); + return _PyLexer_SpanView(tok, token->span, length); } #endif diff --git a/Parser/lexer/lexer_internal.h b/Parser/lexer/lexer_internal.h index c6d3b9045c72921..01825bb0bae3bec 100644 --- a/Parser/lexer/lexer_internal.h +++ b/Parser/lexer/lexer_internal.h @@ -46,6 +46,7 @@ TOK_NEXT_MODE(struct tok_state *tok) int _PyLexer_nextc(struct tok_state *); void _PyLexer_backup(struct tok_state *, int); +void _PyLexer_update_ftstring_expr(struct tok_state *, char); int _PyLexer_set_ftstring_expr(struct tok_state *, struct token *, char); int _PyLexer_check_string_prefixes(struct tok_state *, int, int, int, int, int); int _PyLexer_scan_number(struct tok_state *, struct token *, int, int); diff --git a/Parser/lexer/state.c b/Parser/lexer/state.c index d82a7d0f296bac0..3702e38b8ff74c0 100644 --- a/Parser/lexer/state.c +++ b/Parser/lexer/state.c @@ -60,24 +60,6 @@ _PyTokenizer_tok_new(void) return tok; } -static void -free_fstring_expressions(struct tok_state *tok) -{ - int index; - tokenizer_mode *mode; - - for (index = tok->tok_mode_stack_index; index >= 0; --index) { - mode = &(tok->tok_mode_stack[index]); - if (mode->last_expr_buffer != NULL) { - PyMem_Free(mode->last_expr_buffer); - mode->last_expr_buffer = NULL; - mode->last_expr_size = 0; - mode->last_expr_end = -1; - mode->in_format_spec = 0; - } - } -} - /* Free a tok_state structure */ void _PyTokenizer_Free(struct tok_state *tok) @@ -89,7 +71,6 @@ _PyTokenizer_Free(struct tok_state *tok) Py_XDECREF(tok->module); _PyTok_ReaderFree(tok); _PyTok_SourceClear(&tok->source); - free_fstring_expressions(tok); PyMem_Free(tok); } diff --git a/Parser/lexer/state.h b/Parser/lexer/state.h index 496962fd0484f03..9a2c442898ba388 100644 --- a/Parser/lexer/state.h +++ b/Parser/lexer/state.h @@ -50,16 +50,11 @@ typedef struct _tokenizer_mode { char quote; int quote_size; int raw; - const char* start; - const char* multi_line_start; + _PyTok_Off start; + _PyTok_Off multi_line_start; int first_line; - Py_ssize_t start_offset; - Py_ssize_t multi_line_start_offset; - - Py_ssize_t last_expr_size; - Py_ssize_t last_expr_end; - char* last_expr_buffer; + _PyTok_Span debug_expr; int in_debug; int in_format_spec; diff --git a/Parser/lexer/string.c b/Parser/lexer/string.c index fc0299c5c7c592f..c2c2b50fd053ca8 100644 --- a/Parser/lexer/string.c +++ b/Parser/lexer/string.c @@ -7,6 +7,73 @@ #define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end) +static _PyTok_Off +current_offset(const struct tok_state *tok, const char *position) +{ + assert(position >= tok->buf && position <= tok->inp); + return tok->buf_offset + (position - tok->buf); +} + +static char * +offset_pointer(const struct tok_state *tok, _PyTok_Off offset) +{ + if (offset >= tok->buf_offset && + offset - tok->buf_offset <= tok->inp - tok->buf) { + return tok->buf + (offset - tok->buf_offset); + } + assert(offset >= 0 && offset <= tok->source.len); + return tok->source.bytes + offset; +} + +static Py_ssize_t +strip_expr_comments(const char *expr, Py_ssize_t len, char *result) +{ + Py_ssize_t output = 0; + char quote = 0; + int quote_size = 0; + + for (Py_ssize_t i = 0; i < len;) { + char c = expr[i]; + if (quote != 0) { + if (c == '\\' && i + 1 < len) { + result[output] = c; + result[output + 1] = expr[i + 1]; + output += 2; + i += 2; + continue; + } + if (c == quote) { + if (quote_size == 1) { + quote = 0; + } + else if (i + 2 < len && expr[i + 1] == quote && + expr[i + 2] == quote) { + memcpy(result + output, expr + i, 3); + output += 3; + i += 3; + quote = 0; + continue; + } + } + } + else if (c == '#') { + while (i < len && expr[i] != '\n') { + i++; + } + continue; + } + else if (c == '\'' || c == '"') { + quote = c; + quote_size = i + 2 < len && expr[i + 1] == c && + expr[i + 2] == c ? 3 : 1; + } + result[output] = c; + output++; + i++; + } + return output; +} + int _PyLexer_set_ftstring_expr(struct tok_state* tok, struct token *token, char c) { assert(token != NULL); @@ -16,101 +83,26 @@ _PyLexer_set_ftstring_expr(struct tok_state* tok, struct token *token, char c) { if (!(tok_mode->in_debug || tok_mode->string_kind == TSTRING) || token->metadata) { return 0; } - PyObject *res = NULL; - - // Look for a # character outside of string literals - int hash_detected = 0; - int in_string = 0; - char quote_char = 0; - - for (Py_ssize_t i = 0; i < tok_mode->last_expr_size - tok_mode->last_expr_end; i++) { - char ch = tok_mode->last_expr_buffer[i]; - - // Skip escaped characters - if (ch == '\\') { - i++; - continue; - } - - // Handle quotes - if (ch == '"' || ch == '\'') { - // The following if/else block works becase there is an off number - // of quotes in STRING tokens and the lexer only ever reaches this - // function with valid STRING tokens. - // For example: """hello""" - // First quote: in_string = 1 - // Second quote: in_string = 0 - // Third quote: in_string = 1 - if (!in_string) { - in_string = 1; - quote_char = ch; - } - else if (ch == quote_char) { - in_string = 0; - } - continue; - } - - // Check for # outside strings - if (ch == '#' && !in_string) { - hash_detected = 1; - break; - } + Py_ssize_t expr_len; + const char *expr = _PyLexer_SpanView( + tok, tok_mode->debug_expr, &expr_len); + if (expr == NULL) { + return -1; + } + PyObject *res; + if (memchr(expr, '#', expr_len) == NULL) { + res = PyUnicode_DecodeUTF8(expr, expr_len, NULL); } - // If we found a # character in the expression, we need to handle comments - if (hash_detected) { - // Allocate buffer for processed result - char *result = (char *)PyMem_Malloc((tok_mode->last_expr_size - tok_mode->last_expr_end + 1) * sizeof(char)); - if (!result) { + else { + char *stripped = PyMem_Malloc((size_t)expr_len); + if (stripped == NULL) { + PyErr_NoMemory(); return -1; } - - Py_ssize_t i = 0; // Input position - Py_ssize_t j = 0; // Output position - in_string = 0; // Whether we're in a string - quote_char = 0; // Current string quote char - - // Process each character - while (i < tok_mode->last_expr_size - tok_mode->last_expr_end) { - char ch = tok_mode->last_expr_buffer[i]; - - // Handle string quotes - if (ch == '"' || ch == '\'') { - // See comment above to understand this part - if (!in_string) { - in_string = 1; - quote_char = ch; - } else if (ch == quote_char) { - in_string = 0; - } - result[j++] = ch; - } - // Skip comments - else if (ch == '#' && !in_string) { - while (i < tok_mode->last_expr_size - tok_mode->last_expr_end && - tok_mode->last_expr_buffer[i] != '\n') { - i++; - } - if (i < tok_mode->last_expr_size - tok_mode->last_expr_end) { - result[j++] = '\n'; - } - } - // Copy other chars - else { - result[j++] = ch; - } - i++; - } - - result[j] = '\0'; // Null-terminate the result string - res = PyUnicode_DecodeUTF8(result, j, NULL); - PyMem_Free(result); - } else { - res = PyUnicode_DecodeUTF8( - tok_mode->last_expr_buffer, - tok_mode->last_expr_size - tok_mode->last_expr_end, - NULL - ); + Py_ssize_t stripped_len = strip_expr_comments( + expr, expr_len, stripped); + res = PyUnicode_DecodeUTF8(stripped, stripped_len, NULL); + PyMem_Free(stripped); } if (!res) { @@ -120,61 +112,28 @@ _PyLexer_set_ftstring_expr(struct tok_state* tok, struct token *token, char c) { return 0; } -int +void _PyLexer_update_ftstring_expr(struct tok_state *tok, char cur) { - assert(tok->cur != NULL); - - Py_ssize_t size = cur == 0 - ? tok->inp - tok->cur : (Py_ssize_t)strlen(tok->cur); tokenizer_mode *tok_mode = TOK_GET_MODE(tok); switch (cur) { - case 0: - if (!tok_mode->last_expr_buffer || tok_mode->last_expr_end >= 0) { - return 1; - } - char *new_buffer = PyMem_Realloc( - tok_mode->last_expr_buffer, - tok_mode->last_expr_size + size - ); - if (new_buffer == NULL) { - PyMem_Free(tok_mode->last_expr_buffer); - goto error; - } - tok_mode->last_expr_buffer = new_buffer; - memcpy(tok_mode->last_expr_buffer + tok_mode->last_expr_size, - tok->cur, size); - tok_mode->last_expr_size += size; - break; case '{': - if (tok_mode->last_expr_buffer != NULL) { - PyMem_Free(tok_mode->last_expr_buffer); - } - tok_mode->last_expr_buffer = PyMem_Malloc(size); - if (tok_mode->last_expr_buffer == NULL) { - goto error; - } - tok_mode->last_expr_size = size; - tok_mode->last_expr_end = -1; - memcpy(tok_mode->last_expr_buffer, tok->cur, size); + tok_mode->debug_expr = (_PyTok_Span){ + current_offset(tok, tok->cur), -1}; break; case '}': case '!': - tok_mode->last_expr_end = strlen(tok->start); + tok_mode->debug_expr.end = current_offset(tok, tok->start); break; case ':': - if (tok_mode->last_expr_end == -1) { - tok_mode->last_expr_end = strlen(tok->start); + if (tok_mode->debug_expr.end < 0) { + tok_mode->debug_expr.end = current_offset(tok, tok->start); } break; default: Py_UNREACHABLE(); } - return 1; -error: - tok->done = E_NOMEM; - return 0; } int @@ -265,14 +224,10 @@ _PyLexer_scan_fstring_start(struct tok_state *tok, struct token *token, int c) the_current_tok->kind = TOK_FSTRING_MODE; the_current_tok->quote = quote; the_current_tok->quote_size = quote_size; - the_current_tok->start = tok->start; - the_current_tok->multi_line_start = tok->line_start; + the_current_tok->start = current_offset(tok, tok->start); + the_current_tok->multi_line_start = current_offset(tok, tok->line_start); the_current_tok->first_line = tok->lineno; - the_current_tok->start_offset = -1; - the_current_tok->multi_line_start_offset = -1; - the_current_tok->last_expr_buffer = NULL; - the_current_tok->last_expr_size = 0; - the_current_tok->last_expr_end = -1; + the_current_tok->debug_expr = (_PyTok_Span){-1, -1}; the_current_tok->in_format_spec = 0; the_current_tok->in_debug = 0; @@ -462,13 +417,6 @@ _PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, st } } - if (current_tok->last_expr_buffer != NULL) { - PyMem_Free(current_tok->last_expr_buffer); - current_tok->last_expr_buffer = NULL; - current_tok->last_expr_size = 0; - current_tok->last_expr_end = -1; - } - p_start = tok->start; p_end = tok->cur; tok->tok_mode_stack_index--; @@ -520,9 +468,9 @@ _PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, st // shift the tok_state's location into // the start of string, and report the error // from the initial quote character - tok->cur = (char *)current_tok->start; - tok->cur++; - tok->line_start = current_tok->multi_line_start; + tok->cur = offset_pointer(tok, current_tok->start) + 1; + tok->line_start = offset_pointer( + tok, current_tok->multi_line_start); int start = tok->lineno; tokenizer_mode *the_current_tok = TOK_GET_MODE(tok); @@ -553,9 +501,7 @@ _PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, st } if (c == '{') { - if (!_PyLexer_update_ftstring_expr(tok, c)) { - return MAKE_TOKEN(ENDMARKER); - } + _PyLexer_update_ftstring_expr(tok, c); int peek = tok_nextc(tok); if (peek != '{' || in_format_spec) { tok_backup(tok, peek); diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c index 68c8da2186ede96..c6602286e8248e8 100644 --- a/Parser/tokenizer/reader.c +++ b/Parser/tokenizer/reader.c @@ -672,12 +672,6 @@ _PyTok_ReaderUnderflow(struct tok_state *tok) } tok->implicit_newline = chunk.implicit_newline; - if (!prepared && tok->tok_mode_stack_index && - !_PyLexer_update_ftstring_expr(tok, 0)) { - _PyTok_ChunkClear(&chunk); - tok->input_error = 1; - return 0; - } ADVANCE_LINENO(); if (kind == _PYTOK_READER_FILE && (tok->encoding == NULL || strcmp(tok->encoding, "utf-8") == 0) && From f2bc3ec4953c5d5880047328fa12f3fca5b42263 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Fri, 28 Aug 2026 22:16:00 +0100 Subject: [PATCH 07/11] gh-153569: track f-string comments as source spans --- Lib/test/test_fstring.py | 4 ++ Lib/test/test_tstring.py | 4 ++ Parser/lexer/lexer.c | 12 ++++ Parser/lexer/lexer_internal.h | 2 + Parser/lexer/state.c | 6 ++ Parser/lexer/state.h | 9 +++ Parser/lexer/string.c | 122 +++++++++++++++++++++------------- 7 files changed, 111 insertions(+), 48 deletions(-) diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index debd9a41063e49d..0e4ecfc09c2eb89 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -1686,6 +1686,10 @@ def __repr__(self): self.assertEqual(f'{"""a""""#" # outside =}', '"""a""""#" \n=\'a#\'') + d = {'a#b': 42} + self.assertEqual(f'''{f"{d["a#b"]}"=}''', + 'f"{d["a#b"]}"=\'42\'') + self.assertEqual(f'{ # some comment goes here """hello"""=}', ' \n """hello"""=\'hello\'') self.assertEqual(f'{"""# this is not a comment diff --git a/Lib/test/test_tstring.py b/Lib/test/test_tstring.py index b60b50e446e3656..78e20187440ec51 100644 --- a/Lib/test/test_tstring.py +++ b/Lib/test/test_tstring.py @@ -295,5 +295,9 @@ def test_triple_quoted(self): }' self.assertEqual(t.interpolations[0].expression, '"""a""""#"') + d = {'a#b': 42} + t = t'''{f"{d["a#b"]}"}''' + self.assertEqual(t.interpolations[0].expression, 'f"{d["a#b"]}"') + if __name__ == '__main__': unittest.main() diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index 79b4de007c6daf1..44e088066ba6ea0 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -332,6 +332,18 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str c = tok_nextc(tok); } + if (INSIDE_FSTRING(tok) && INSIDE_FSTRING_EXPR(current_tok)) { + const char *comment_end = tok->cur; + if (c == '\n') { + comment_end--; + } + if (_PyLexer_record_ftstring_comment( + tok, tok->start, comment_end) < 0) { + tok->done = E_NOMEM; + return MAKE_TOKEN(ERRORTOKEN); + } + } + if (tok->tok_extra_tokens) { p = tok->start; } diff --git a/Parser/lexer/lexer_internal.h b/Parser/lexer/lexer_internal.h index 01825bb0bae3bec..dd9e445b646379f 100644 --- a/Parser/lexer/lexer_internal.h +++ b/Parser/lexer/lexer_internal.h @@ -47,6 +47,8 @@ TOK_NEXT_MODE(struct tok_state *tok) int _PyLexer_nextc(struct tok_state *); void _PyLexer_backup(struct tok_state *, int); void _PyLexer_update_ftstring_expr(struct tok_state *, char); +int _PyLexer_record_ftstring_comment( + struct tok_state *, const char *, const char *); int _PyLexer_set_ftstring_expr(struct tok_state *, struct token *, char); int _PyLexer_check_string_prefixes(struct tok_state *, int, int, int, int, int); int _PyLexer_scan_number(struct tok_state *, struct token *, int, int); diff --git a/Parser/lexer/state.c b/Parser/lexer/state.c index 3702e38b8ff74c0..d9ebe8f60940b2f 100644 --- a/Parser/lexer/state.c +++ b/Parser/lexer/state.c @@ -71,6 +71,12 @@ _PyTokenizer_Free(struct tok_state *tok) Py_XDECREF(tok->module); _PyTok_ReaderFree(tok); _PyTok_SourceClear(&tok->source); + tokenizer_comments *comments = tok->ftstring_comments; + while (comments != NULL) { + tokenizer_comments *previous = comments->previous; + PyMem_Free(comments); + comments = previous; + } PyMem_Free(tok); } diff --git a/Parser/lexer/state.h b/Parser/lexer/state.h index 9a2c442898ba388..3605db001ed4824 100644 --- a/Parser/lexer/state.h +++ b/Parser/lexer/state.h @@ -61,6 +61,14 @@ typedef struct _tokenizer_mode { enum string_kind_t string_kind; } tokenizer_mode; +typedef struct _tokenizer_comments { + struct _tokenizer_comments *previous; + Py_ssize_t count; + Py_ssize_t capacity; + int mode; + _PyTok_Span spans[]; +} tokenizer_comments; + /* Tokenizer state */ struct tok_state { /* Input state; buf <= cur <= inp */ @@ -116,6 +124,7 @@ struct tok_state { // TODO: Factor this into its own thing tokenizer_mode tok_mode_stack[MAXFSTRINGLEVEL]; int tok_mode_stack_index; + tokenizer_comments *ftstring_comments; int tok_extra_tokens; int comment_newline; int implicit_newline; diff --git a/Parser/lexer/string.c b/Parser/lexer/string.c index c2c2b50fd053ca8..6839d1b6e0bce7f 100644 --- a/Parser/lexer/string.c +++ b/Parser/lexer/string.c @@ -25,53 +25,53 @@ offset_pointer(const struct tok_state *tok, _PyTok_Off offset) return tok->source.bytes + offset; } -static Py_ssize_t -strip_expr_comments(const char *expr, Py_ssize_t len, char *result) +static tokenizer_comments * +current_comments(const struct tok_state *tok) { - Py_ssize_t output = 0; - char quote = 0; - int quote_size = 0; - - for (Py_ssize_t i = 0; i < len;) { - char c = expr[i]; - if (quote != 0) { - if (c == '\\' && i + 1 < len) { - result[output] = c; - result[output + 1] = expr[i + 1]; - output += 2; - i += 2; - continue; - } - if (c == quote) { - if (quote_size == 1) { - quote = 0; - } - else if (i + 2 < len && expr[i + 1] == quote && - expr[i + 2] == quote) { - memcpy(result + output, expr + i, 3); - output += 3; - i += 3; - quote = 0; - continue; - } - } + tokenizer_comments *comments = tok->ftstring_comments; + return comments != NULL && comments->mode == tok->tok_mode_stack_index + ? comments : NULL; +} + +int +_PyLexer_record_ftstring_comment(struct tok_state *tok, const char *start, + const char *end) +{ + tokenizer_mode *mode = TOK_GET_MODE(tok); + if (mode->debug_expr.end >= 0) { + return 0; + } + assert(mode->debug_expr.start >= 0); + tokenizer_comments *comments = current_comments(tok); + if (comments == NULL || comments->count == comments->capacity) { + int create = comments == NULL; + Py_ssize_t max_capacity = (PY_SSIZE_T_MAX - + (Py_ssize_t)sizeof(*comments)) / + (Py_ssize_t)sizeof(*comments->spans); + if (comments != NULL && comments->capacity > max_capacity / 2) { + PyErr_NoMemory(); + return -1; } - else if (c == '#') { - while (i < len && expr[i] != '\n') { - i++; - } - continue; + Py_ssize_t capacity = comments == NULL ? 4 : comments->capacity * 2; + size_t size = sizeof(*comments) + + (size_t)capacity * sizeof(*comments->spans); + tokenizer_comments *resized = PyMem_Realloc(comments, size); + if (resized == NULL) { + PyErr_NoMemory(); + return -1; } - else if (c == '\'' || c == '"') { - quote = c; - quote_size = i + 2 < len && expr[i + 1] == c && - expr[i + 2] == c ? 3 : 1; + comments = resized; + if (create) { + comments->previous = tok->ftstring_comments; + comments->count = 0; + comments->mode = tok->tok_mode_stack_index; } - result[output] = c; - output++; - i++; + comments->capacity = capacity; + tok->ftstring_comments = comments; } - return output; + comments->spans[comments->count++] = (_PyTok_Span){ + current_offset(tok, start), current_offset(tok, end)}; + return 0; } int @@ -89,21 +89,38 @@ _PyLexer_set_ftstring_expr(struct tok_state* tok, struct token *token, char c) { if (expr == NULL) { return -1; } + tokenizer_comments *comments = current_comments(tok); PyObject *res; - if (memchr(expr, '#', expr_len) == NULL) { - res = PyUnicode_DecodeUTF8(expr, expr_len, NULL); - } - else { + if (comments != NULL && comments->count > 0) { char *stripped = PyMem_Malloc((size_t)expr_len); if (stripped == NULL) { PyErr_NoMemory(); return -1; } - Py_ssize_t stripped_len = strip_expr_comments( - expr, expr_len, stripped); + _PyTok_Off copied_to = tok_mode->debug_expr.start; + Py_ssize_t stripped_len = 0; + for (Py_ssize_t i = 0; i < comments->count; i++) { + _PyTok_Span comment = comments->spans[i]; + assert(comment.start >= copied_to); + assert(comment.end <= tok_mode->debug_expr.end); + Py_ssize_t length = comment.start - copied_to; + memcpy(stripped + stripped_len, + expr + copied_to - tok_mode->debug_expr.start, + (size_t)length); + stripped_len += length; + copied_to = comment.end; + } + Py_ssize_t length = tok_mode->debug_expr.end - copied_to; + memcpy(stripped + stripped_len, + expr + copied_to - tok_mode->debug_expr.start, + (size_t)length); + stripped_len += length; res = PyUnicode_DecodeUTF8(stripped, stripped_len, NULL); PyMem_Free(stripped); } + else { + res = PyUnicode_DecodeUTF8(expr, expr_len, NULL); + } if (!res) { return -1; @@ -121,6 +138,10 @@ _PyLexer_update_ftstring_expr(struct tok_state *tok, char cur) case '{': tok_mode->debug_expr = (_PyTok_Span){ current_offset(tok, tok->cur), -1}; + tokenizer_comments *comments = current_comments(tok); + if (comments != NULL) { + comments->count = 0; + } break; case '}': case '!': @@ -419,6 +440,11 @@ _PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, st p_start = tok->start; p_end = tok->cur; + tokenizer_comments *comments = current_comments(tok); + if (comments != NULL) { + tok->ftstring_comments = comments->previous; + PyMem_Free(comments); + } tok->tok_mode_stack_index--; return MAKE_TOKEN(FTSTRING_END(current_tok)); From fa4714d7fba790b567d4641d1f788f9d029e9dd6 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Fri, 28 Aug 2026 23:43:46 +0100 Subject: [PATCH 08/11] gh-153569: simplify f-string span handling --- Parser/lexer/lexer.c | 3 +- Parser/lexer/lexer.h | 23 +++------- Parser/lexer/lexer_internal.h | 2 +- Parser/lexer/state.c | 22 +--------- Parser/lexer/state.h | 30 ++++++++++++- Parser/lexer/string.c | 80 +++++++++++++++++++++-------------- 6 files changed, 86 insertions(+), 74 deletions(-) diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index 44e088066ba6ea0..154af82fe13f883 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -570,7 +570,8 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str if (cursor_valid) { _PyLexer_update_ftstring_expr(tok, c); } - if ((cursor_valid) && c != '{' && _PyLexer_set_ftstring_expr(tok, token, c)) { + if (cursor_valid && c != '{' && + _PyLexer_set_ftstring_expr_metadata(tok, token)) { return MAKE_TOKEN(ERRORTOKEN); } diff --git a/Parser/lexer/lexer.h b/Parser/lexer/lexer.h index 776504c7c186777..8c63b3b0a3043df 100644 --- a/Parser/lexer/lexer.h +++ b/Parser/lexer/lexer.h @@ -5,22 +5,6 @@ int _PyTokenizer_Get(struct tok_state *, struct token *); -static inline const char * -_PyLexer_SpanView(const struct tok_state *tok, _PyTok_Span span, - Py_ssize_t *length) -{ - assert(length != NULL); - assert(_PyTok_SpanIsValid(span)); - assert(tok->buf != NULL); - assert(tok->inp >= tok->buf); - if (span.start >= tok->buf_offset && - span.end - tok->buf_offset <= tok->inp - tok->buf) { - *length = span.end - span.start; - return tok->buf + (span.start - tok->buf_offset); - } - return _PyTok_SourceSpanView(&tok->source, span, length); -} - /* The view points into the current input window. The next _PyTokenizer_Get() call may discard it. */ static inline const char * @@ -33,7 +17,12 @@ _PyToken_TextView(const struct tok_state *tok, const struct token *token, *length = 0; return ""; } - return _PyLexer_SpanView(tok, token->span, length); + assert(tok->buf != NULL); + assert(tok->inp >= tok->buf); + assert(token->span.start >= tok->buf_offset); + assert(token->span.end - tok->buf_offset <= tok->inp - tok->buf); + *length = token->span.end - token->span.start; + return tok->buf + (token->span.start - tok->buf_offset); } #endif diff --git a/Parser/lexer/lexer_internal.h b/Parser/lexer/lexer_internal.h index dd9e445b646379f..9dc70b1e6a42801 100644 --- a/Parser/lexer/lexer_internal.h +++ b/Parser/lexer/lexer_internal.h @@ -49,7 +49,7 @@ void _PyLexer_backup(struct tok_state *, int); void _PyLexer_update_ftstring_expr(struct tok_state *, char); int _PyLexer_record_ftstring_comment( struct tok_state *, const char *, const char *); -int _PyLexer_set_ftstring_expr(struct tok_state *, struct token *, char); +int _PyLexer_set_ftstring_expr_metadata(struct tok_state *, struct token *); int _PyLexer_check_string_prefixes(struct tok_state *, int, int, int, int, int); int _PyLexer_scan_number(struct tok_state *, struct token *, int, int); int _PyLexer_scan_fstring_start(struct tok_state *, struct token *, int); diff --git a/Parser/lexer/state.c b/Parser/lexer/state.c index d9ebe8f60940b2f..11bfd31fe328443 100644 --- a/Parser/lexer/state.c +++ b/Parser/lexer/state.c @@ -95,31 +95,11 @@ _PyToken_Init(struct token *token) { token->metadata = NULL; } -static inline _PyTok_Span -buffer_span(const struct tok_state *tok, const char *start, const char *end) -{ - if (start == NULL) { - assert(end == NULL); - return (_PyTok_Span){-1, -1}; - } - assert(end != NULL); - const char *base = tok->buf; - assert(base != NULL); - assert(tok->inp >= base); - Py_ssize_t start_offset = start - base; - Py_ssize_t end_offset = end - base; - assert(start_offset >= 0 && start_offset <= end_offset); - assert(end_offset <= tok->inp - base); - assert(tok->buf_offset <= PY_SSIZE_T_MAX - end_offset); - return _PyTok_SpanFromBounds( - tok->buf_offset + start_offset, tok->buf_offset + end_offset); -} - int _PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, const char *start, const char *end) { token->level = tok->level; - token->span = buffer_span(tok, start, end); + token->span = _PyLexer_BufferSpan(tok, start, end); int lineno = ISSTRINGLIT(type) ? tok->first_lineno : tok->lineno; token->start_loc = (_PyTok_Loc){lineno, -1}; token->end_loc = (_PyTok_Loc){tok->lineno, -1}; diff --git a/Parser/lexer/state.h b/Parser/lexer/state.h index 3605db001ed4824..c62c681da41275a 100644 --- a/Parser/lexer/state.h +++ b/Parser/lexer/state.h @@ -54,7 +54,7 @@ typedef struct _tokenizer_mode { _PyTok_Off multi_line_start; int first_line; - _PyTok_Span debug_expr; + _PyTok_Span expr_span; int in_debug; int in_format_spec; @@ -65,7 +65,7 @@ typedef struct _tokenizer_comments { struct _tokenizer_comments *previous; Py_ssize_t count; Py_ssize_t capacity; - int mode; + int mode_index; _PyTok_Span spans[]; } tokenizer_comments; @@ -133,6 +133,32 @@ struct tok_state { #endif }; +static inline _PyTok_Off +_PyLexer_BufferOffset(const struct tok_state *tok, const char *position) +{ + assert(tok->buf != NULL); + assert(tok->inp >= tok->buf); + assert(position >= tok->buf && position <= tok->inp); + Py_ssize_t offset = position - tok->buf; + assert(tok->buf_offset <= PY_SSIZE_T_MAX - offset); + return tok->buf_offset + offset; +} + +static inline _PyTok_Span +_PyLexer_BufferSpan(const struct tok_state *tok, const char *start, + const char *end) +{ + if (start == NULL) { + assert(end == NULL); + return (_PyTok_Span){-1, -1}; + } + assert(end != NULL); + assert(start <= end); + return _PyTok_SpanFromBounds( + _PyLexer_BufferOffset(tok, start), + _PyLexer_BufferOffset(tok, end)); +} + int _PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, const char *start, const char *end); struct tok_state *_PyTokenizer_tok_new(void); diff --git a/Parser/lexer/string.c b/Parser/lexer/string.c index 6839d1b6e0bce7f..6e498696dd5ff7c 100644 --- a/Parser/lexer/string.c +++ b/Parser/lexer/string.c @@ -7,13 +7,6 @@ #define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end) -static _PyTok_Off -current_offset(const struct tok_state *tok, const char *position) -{ - assert(position >= tok->buf && position <= tok->inp); - return tok->buf_offset + (position - tok->buf); -} - static char * offset_pointer(const struct tok_state *tok, _PyTok_Off offset) { @@ -25,11 +18,25 @@ offset_pointer(const struct tok_state *tok, _PyTok_Off offset) return tok->source.bytes + offset; } +static const char * +span_view(const struct tok_state *tok, _PyTok_Span span, Py_ssize_t *length) +{ + assert(length != NULL); + assert(_PyTok_SpanIsValid(span)); + if (span.start >= tok->buf_offset && + span.end - tok->buf_offset <= tok->inp - tok->buf) { + *length = span.end - span.start; + return tok->buf + (span.start - tok->buf_offset); + } + return _PyTok_SourceSpanView(&tok->source, span, length); +} + static tokenizer_comments * current_comments(const struct tok_state *tok) { tokenizer_comments *comments = tok->ftstring_comments; - return comments != NULL && comments->mode == tok->tok_mode_stack_index + return comments != NULL && + comments->mode_index == tok->tok_mode_stack_index ? comments : NULL; } @@ -38,10 +45,10 @@ _PyLexer_record_ftstring_comment(struct tok_state *tok, const char *start, const char *end) { tokenizer_mode *mode = TOK_GET_MODE(tok); - if (mode->debug_expr.end >= 0) { + if (mode->expr_span.end >= 0) { return 0; } - assert(mode->debug_expr.start >= 0); + assert(mode->expr_span.start >= 0); tokenizer_comments *comments = current_comments(tok); if (comments == NULL || comments->count == comments->capacity) { int create = comments == NULL; @@ -64,55 +71,62 @@ _PyLexer_record_ftstring_comment(struct tok_state *tok, const char *start, if (create) { comments->previous = tok->ftstring_comments; comments->count = 0; - comments->mode = tok->tok_mode_stack_index; + comments->mode_index = tok->tok_mode_stack_index; } comments->capacity = capacity; tok->ftstring_comments = comments; } - comments->spans[comments->count++] = (_PyTok_Span){ - current_offset(tok, start), current_offset(tok, end)}; + comments->spans[comments->count++] = + _PyLexer_BufferSpan(tok, start, end); return 0; } int -_PyLexer_set_ftstring_expr(struct tok_state* tok, struct token *token, char c) { +_PyLexer_set_ftstring_expr_metadata(struct tok_state *tok, struct token *token) +{ assert(token != NULL); - assert(c == '}' || c == ':' || c == '!'); tokenizer_mode *tok_mode = TOK_GET_MODE(tok); if (!(tok_mode->in_debug || tok_mode->string_kind == TSTRING) || token->metadata) { return 0; } Py_ssize_t expr_len; - const char *expr = _PyLexer_SpanView( - tok, tok_mode->debug_expr, &expr_len); + const char *expr = span_view(tok, tok_mode->expr_span, &expr_len); if (expr == NULL) { return -1; } tokenizer_comments *comments = current_comments(tok); PyObject *res; if (comments != NULL && comments->count > 0) { - char *stripped = PyMem_Malloc((size_t)expr_len); + Py_ssize_t stripped_size = expr_len; + _PyTok_Off previous_end = tok_mode->expr_span.start; + for (Py_ssize_t i = 0; i < comments->count; i++) { + _PyTok_Span comment = comments->spans[i]; + assert(_PyTok_SpanIsValid(comment)); + assert(comment.start >= previous_end); + assert(comment.end <= tok_mode->expr_span.end); + stripped_size -= comment.end - comment.start; + previous_end = comment.end; + } + char *stripped = PyMem_Malloc((size_t)stripped_size); if (stripped == NULL) { PyErr_NoMemory(); return -1; } - _PyTok_Off copied_to = tok_mode->debug_expr.start; + _PyTok_Off copied_to = tok_mode->expr_span.start; Py_ssize_t stripped_len = 0; for (Py_ssize_t i = 0; i < comments->count; i++) { _PyTok_Span comment = comments->spans[i]; - assert(comment.start >= copied_to); - assert(comment.end <= tok_mode->debug_expr.end); Py_ssize_t length = comment.start - copied_to; memcpy(stripped + stripped_len, - expr + copied_to - tok_mode->debug_expr.start, + expr + copied_to - tok_mode->expr_span.start, (size_t)length); stripped_len += length; copied_to = comment.end; } - Py_ssize_t length = tok_mode->debug_expr.end - copied_to; + Py_ssize_t length = tok_mode->expr_span.end - copied_to; memcpy(stripped + stripped_len, - expr + copied_to - tok_mode->debug_expr.start, + expr + copied_to - tok_mode->expr_span.start, (size_t)length); stripped_len += length; res = PyUnicode_DecodeUTF8(stripped, stripped_len, NULL); @@ -136,8 +150,8 @@ _PyLexer_update_ftstring_expr(struct tok_state *tok, char cur) switch (cur) { case '{': - tok_mode->debug_expr = (_PyTok_Span){ - current_offset(tok, tok->cur), -1}; + tok_mode->expr_span = (_PyTok_Span){ + _PyLexer_BufferOffset(tok, tok->cur), -1}; tokenizer_comments *comments = current_comments(tok); if (comments != NULL) { comments->count = 0; @@ -145,11 +159,12 @@ _PyLexer_update_ftstring_expr(struct tok_state *tok, char cur) break; case '}': case '!': - tok_mode->debug_expr.end = current_offset(tok, tok->start); + tok_mode->expr_span.end = _PyLexer_BufferOffset(tok, tok->start); break; case ':': - if (tok_mode->debug_expr.end < 0) { - tok_mode->debug_expr.end = current_offset(tok, tok->start); + if (tok_mode->expr_span.end < 0) { + tok_mode->expr_span.end = + _PyLexer_BufferOffset(tok, tok->start); } break; default: @@ -245,10 +260,11 @@ _PyLexer_scan_fstring_start(struct tok_state *tok, struct token *token, int c) the_current_tok->kind = TOK_FSTRING_MODE; the_current_tok->quote = quote; the_current_tok->quote_size = quote_size; - the_current_tok->start = current_offset(tok, tok->start); - the_current_tok->multi_line_start = current_offset(tok, tok->line_start); + the_current_tok->start = _PyLexer_BufferOffset(tok, tok->start); + the_current_tok->multi_line_start = + _PyLexer_BufferOffset(tok, tok->line_start); the_current_tok->first_line = tok->lineno; - the_current_tok->debug_expr = (_PyTok_Span){-1, -1}; + the_current_tok->expr_span = (_PyTok_Span){-1, -1}; the_current_tok->in_format_spec = 0; the_current_tok->in_debug = 0; From 049f228cb1c67b8fadd6f937ed349bc8396d3bd1 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sat, 29 Aug 2026 00:33:22 +0100 Subject: [PATCH 09/11] gh-153569: keep token span invariants explicit --- Parser/lexer/lexer.h | 1 + 1 file changed, 1 insertion(+) diff --git a/Parser/lexer/lexer.h b/Parser/lexer/lexer.h index 8c63b3b0a3043df..a82add64263e2e5 100644 --- a/Parser/lexer/lexer.h +++ b/Parser/lexer/lexer.h @@ -17,6 +17,7 @@ _PyToken_TextView(const struct tok_state *tok, const struct token *token, *length = 0; return ""; } + assert(_PyTok_SpanIsValid(token->span)); assert(tok->buf != NULL); assert(tok->inp >= tok->buf); assert(token->span.start >= tok->buf_offset); From eae3e331528bf50c8d0e96a973040a1f0f3c0ddc Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sat, 29 Aug 2026 00:40:18 +0100 Subject: [PATCH 10/11] gh-153569: preserve comments after not-equal expressions --- Lib/test/test_fstring.py | 4 ++++ Lib/test/test_tstring.py | 5 +++++ Parser/lexer/lexer.c | 7 +++++++ 3 files changed, 16 insertions(+) diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index 0e4ecfc09c2eb89..ae443b269fe4dde 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -1686,6 +1686,10 @@ def __repr__(self): self.assertEqual(f'{"""a""""#" # outside =}', '"""a""""#" \n=\'a#\'') + x, y = 1, 2 + self.assertEqual(f'{x != y # outside +=}', 'x != y \n=True') + d = {'a#b': 42} self.assertEqual(f'''{f"{d["a#b"]}"=}''', 'f"{d["a#b"]}"=\'42\'') diff --git a/Lib/test/test_tstring.py b/Lib/test/test_tstring.py index 78e20187440ec51..c90f18a4ced5296 100644 --- a/Lib/test/test_tstring.py +++ b/Lib/test/test_tstring.py @@ -295,6 +295,11 @@ def test_triple_quoted(self): }' self.assertEqual(t.interpolations[0].expression, '"""a""""#"') + x, y = 1, 2 + t = t'{x != y # outside +}' + self.assertEqual(t.interpolations[0].expression, 'x != y') + d = {'a#b': 42} t = t'''{f"{d["a#b"]}"}''' self.assertEqual(t.interpolations[0].expression, 'f"{d["a#b"]}"') diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index 154af82fe13f883..f45ed36736c3638 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -567,6 +567,13 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str int cursor_in_format_with_debug = cursor == 1 && (current_tok->in_debug || in_format_spec); int cursor_valid = cursor == 0 || cursor_in_format_with_debug; + if (cursor_valid && c == '!') { + int c2 = tok_nextc(tok); + if (c2 == '=') { + cursor_valid = 0; + } + tok_backup(tok, c2); + } if (cursor_valid) { _PyLexer_update_ftstring_expr(tok, c); } From 3ce81ac9da1d824b630ed1a1938660537d90e127 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sat, 29 Aug 2026 02:16:55 +0100 Subject: [PATCH 11/11] gh-153569: remove duplicated tokenizer state --- Makefile.pre.in | 2 - PCbuild/_freeze_module.vcxproj | 1 - PCbuild/_freeze_module.vcxproj.filters | 3 - PCbuild/pythoncore.vcxproj | 2 - PCbuild/pythoncore.vcxproj.filters | 6 -- Parser/lexer/buffer.c | 33 -------- Parser/lexer/buffer.h | 22 ------ Parser/lexer/lexer.c | 35 +++++---- Parser/lexer/lexer_internal.h | 8 ++ Parser/lexer/state.c | 26 ++----- Parser/lexer/state.h | 41 +++------- Parser/lexer/string.c | 72 ++++++----------- Parser/pegen.c | 24 ++---- Parser/pegen_errors.c | 46 ++++------- Parser/tokenizer/decoder.c | 3 +- Parser/tokenizer/helpers.c | 8 -- Parser/tokenizer/helpers.h | 4 - Parser/tokenizer/reader.c | 104 ++++++++++++++++--------- Parser/tokenizer/reader.h | 2 + Parser/tokenizer/reader_internal.h | 2 + Parser/tokenizer/source.c | 2 +- Parser/tokenizer/source.h | 6 ++ Python/Python-tokenize.c | 3 +- Tools/peg_generator/pegen/build.py | 1 - 24 files changed, 172 insertions(+), 284 deletions(-) delete mode 100644 Parser/lexer/buffer.c delete mode 100644 Parser/lexer/buffer.h diff --git a/Makefile.pre.in b/Makefile.pre.in index 982b2d66216c982..50f4b6770c9d7e2 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -394,7 +394,6 @@ PEGEN_OBJS= \ Parser/peg_api.o TOKENIZER_OBJS= \ - Parser/lexer/buffer.o \ Parser/lexer/lexer.o \ Parser/lexer/number.o \ Parser/lexer/state.o \ @@ -410,7 +409,6 @@ PEGEN_HEADERS= \ $(srcdir)/Parser/string_parser.h TOKENIZER_HEADERS= \ - Parser/lexer/buffer.h \ Parser/lexer/lexer.h \ Parser/lexer/lexer_internal.h \ Parser/lexer/state.h \ diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 469fd77cc8be9dc..a6a37d7be9608f3 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -181,7 +181,6 @@ - diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 976c99b7d24bdfd..27d47ba14e6c2a0 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -469,9 +469,6 @@ Source Files - - Source Files - Source Files diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 93dd56a8ef166f6..055fab2d6357c34 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -422,7 +422,6 @@ - @@ -591,7 +590,6 @@ - diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index f97b51a0e48a3eb..ceec2cd736710aa 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -330,9 +330,6 @@ Parser - - Parser - Parser @@ -1355,9 +1352,6 @@ Parser - - Parser - Parser diff --git a/Parser/lexer/buffer.c b/Parser/lexer/buffer.c deleted file mode 100644 index 7e2330482ec85dc..000000000000000 --- a/Parser/lexer/buffer.c +++ /dev/null @@ -1,33 +0,0 @@ -#include "Python.h" -#include "buffer.h" -#include "state.h" - -void -_PyLexer_SaveBufferPointers(struct tok_state *tok, const char *base, - _PyLexer_BufferPointers *pointers) -{ - pointers->buf_from_base = tok->buf - base; - pointers->cur_from_buf = tok->cur - tok->buf; - pointers->inp_from_buf = tok->inp - tok->buf; - pointers->start_from_buf = tok->start == NULL - ? -1 : tok->start - tok->buf; - pointers->line_start_from_buf = tok->line_start == NULL - ? -1 : tok->line_start - tok->buf; - pointers->multi_line_start_from_buf = tok->multi_line_start == NULL - ? -1 : tok->multi_line_start - tok->buf; -} - -void -_PyLexer_RestoreBufferPointers(struct tok_state *tok, char *base, - const _PyLexer_BufferPointers *pointers) -{ - tok->buf = base + pointers->buf_from_base; - tok->cur = tok->buf + pointers->cur_from_buf; - tok->inp = tok->buf + pointers->inp_from_buf; - tok->start = pointers->start_from_buf < 0 - ? NULL : tok->buf + pointers->start_from_buf; - tok->line_start = pointers->line_start_from_buf < 0 - ? NULL : tok->buf + pointers->line_start_from_buf; - tok->multi_line_start = pointers->multi_line_start_from_buf < 0 - ? NULL : tok->buf + pointers->multi_line_start_from_buf; -} diff --git a/Parser/lexer/buffer.h b/Parser/lexer/buffer.h deleted file mode 100644 index 285da124226d50e..000000000000000 --- a/Parser/lexer/buffer.h +++ /dev/null @@ -1,22 +0,0 @@ -#ifndef _LEXER_BUFFER_H_ -#define _LEXER_BUFFER_H_ - -#include "pyport.h" - -struct tok_state; - -typedef struct { - Py_ssize_t buf_from_base; - Py_ssize_t cur_from_buf; - Py_ssize_t inp_from_buf; - Py_ssize_t start_from_buf; - Py_ssize_t line_start_from_buf; - Py_ssize_t multi_line_start_from_buf; -} _PyLexer_BufferPointers; - -void _PyLexer_SaveBufferPointers( - struct tok_state *, const char *, _PyLexer_BufferPointers *); -void _PyLexer_RestoreBufferPointers( - struct tok_state *, char *, const _PyLexer_BufferPointers *); - -#endif diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index f45ed36736c3638..ffc858e8da829ca 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -7,7 +7,7 @@ #include "../tokenizer/helpers.h" #include "../tokenizer/reader.h" -/* Alternate tab spacing */ +#define TABSIZE 8 #define ALTTABSIZE 1 @@ -45,11 +45,10 @@ _PyLexer_nextc(struct tok_state *tok) int rc; for (;;) { if (tok->cur != tok->inp) { - if ((unsigned int) tok->col_offset >= (unsigned int) INT_MAX) { + if (tok->cur - tok->line_start >= INT_MAX) { tok->done = E_COLUMNOVERFLOW; return EOF; } - tok->col_offset++; return Py_CHARMASK(*tok->cur++); /* Fast path */ } if (tok->done != E_OK) { @@ -89,7 +88,6 @@ _PyLexer_backup(struct tok_state *tok, int c) if ((int)(unsigned char)*tok->cur != Py_CHARMASK(c)) { Py_FatalError("tok_backup: wrong character"); } - tok->col_offset--; } } @@ -103,7 +101,7 @@ verify_identifier(struct tok_state *tok) return 1; } PyObject *s; - if (tok->input_error) + if (tok_failed(tok)) return 0; s = PyUnicode_DecodeUTF8(tok->start, tok->cur - tok->start, NULL); if (s == NULL) { @@ -180,7 +178,7 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str const char *p_end = NULL; nextline: tok->start = NULL; - tok->starting_col_offset = -1; + tok->start_loc = (_PyTok_Loc){tok->lineno, -1}; blankline = 0; @@ -196,7 +194,7 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str col++, altcol++; } else if (c == '\t') { - col = (col / tok->tabsize + 1) * tok->tabsize; + col = (col / TABSIZE + 1) * TABSIZE; altcol = (altcol / ALTTABSIZE + 1) * ALTTABSIZE; } else if (c == '\014') {/* Control-L (formfeed) */ @@ -221,15 +219,16 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str } tok_backup(tok, c); if (c == '#' || c == '\n' || c == '\r') { + int interactive = _PyTok_ReaderIsInteractive(tok); /* Lines with only whitespace and/or comments shouldn't affect the indentation and are not passed to the parser as NEWLINE tokens, except *totally* empty lines in interactive mode, which signal the end of a command group. */ - if (col == 0 && c == '\n' && tok->prompt != NULL) { + if (col == 0 && c == '\n' && interactive) { blankline = 0; /* Let it through */ } - else if (tok->prompt != NULL && tok->lineno == 1) { + else if (interactive && tok->lineno == 1) { /* In interactive mode, if the first line contains only spaces and/or a comment, let it through. */ blankline = 0; @@ -284,7 +283,8 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str } tok->start = tok->cur; - tok->starting_col_offset = tok->col_offset; + tok->start_loc = (_PyTok_Loc){ + tok->lineno, tok->cur != NULL ? _PyLexer_ByteColumn(tok) : -1}; /* Return pending indents/dedents */ if (tok->pendin != 0) { @@ -319,7 +319,8 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str /* Set start of current token */ tok->start = tok->cur == NULL ? NULL : tok->cur - 1; - tok->starting_col_offset = tok->col_offset - 1; + tok->start_loc = (_PyTok_Loc){ + tok->lineno, tok->cur != NULL ? _PyLexer_ByteColumn(tok) - 1 : -1}; /* Skip comment, unless it's a type comment */ if (c == '#') { @@ -350,7 +351,7 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str if (tok->type_comments) { p = tok->start; - current_starting_col_offset = tok->starting_col_offset; + current_starting_col_offset = tok->start_loc.byte_col; prefix = type_comment_prefix; while (*prefix && p < tok->cur) { if (*prefix == ' ') { @@ -394,11 +395,15 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str tok_nextc(tok); tok->atbol = 1; } - return MAKE_TYPE_COMMENT_TOKEN(TYPE_IGNORE, ignore_end_col_offset, tok->col_offset); + return MAKE_TYPE_COMMENT_TOKEN( + TYPE_IGNORE, ignore_end_col_offset, + _PyLexer_ByteColumn(tok)); } else { p_start = type_start; p_end = tok->cur; - return MAKE_TYPE_COMMENT_TOKEN(TYPE_COMMENT, current_starting_col_offset, tok->col_offset); + return MAKE_TYPE_COMMENT_TOKEN( + TYPE_COMMENT, current_starting_col_offset, + _PyLexer_ByteColumn(tok)); } } } @@ -719,7 +724,7 @@ int _PyTokenizer_Get(struct tok_state *tok, struct token *token) { int result = tok_get(tok, token); - if (tok->input_error) { + if (tok_failed(tok)) { result = ERRORTOKEN; } return result; diff --git a/Parser/lexer/lexer_internal.h b/Parser/lexer/lexer_internal.h index 9dc70b1e6a42801..68273eb430e6812 100644 --- a/Parser/lexer/lexer_internal.h +++ b/Parser/lexer/lexer_internal.h @@ -1,6 +1,7 @@ #ifndef _PY_LEXER_INTERNAL_H_ #define _PY_LEXER_INTERNAL_H_ +#include "errcode.h" #include "lexer.h" #define is_potential_identifier_start(c) (\ @@ -44,6 +45,13 @@ TOK_NEXT_MODE(struct tok_state *tok) #define tok_nextc _PyLexer_nextc #define tok_backup _PyLexer_backup +static inline int +tok_failed(const struct tok_state *tok) +{ + return tok->done != E_OK && tok->done != E_EOF && + tok->done != E_INTERACT_STOP; +} + int _PyLexer_nextc(struct tok_state *); void _PyLexer_backup(struct tok_state *, int); void _PyLexer_update_ftstring_expr(struct tok_state *, char); diff --git a/Parser/lexer/state.c b/Parser/lexer/state.c index 11bfd31fe328443..809df08d0d29738 100644 --- a/Parser/lexer/state.c +++ b/Parser/lexer/state.c @@ -6,9 +6,6 @@ #include "state.h" #include "../tokenizer/reader.h" -/* Never change this */ -#define TABSIZE 8 - /* Create and initialize a new tok_state structure */ struct tok_state * _PyTokenizer_tok_new(void) @@ -22,31 +19,21 @@ _PyTokenizer_tok_new(void) } tok->buf = tok->cur = tok->inp = NULL; - tok->fp_interactive = 0; - tok->interactive_src_start = NULL; - tok->interactive_src_end = NULL; tok->start = NULL; tok->done = E_OK; tok->fp = NULL; - tok->tabsize = TABSIZE; tok->indent = 0; tok->indstack[0] = 0; tok->atbol = 1; tok->pendin = 0; - tok->prompt = NULL; tok->lineno = 0; - tok->starting_col_offset = -1; - tok->col_offset = -1; + tok->start_loc = (_PyTok_Loc){-1, -1}; tok->level = 0; tok->altindstack[0] = 0; - tok->input_error = 0; tok->encoding = NULL; tok->filename = NULL; tok->module = NULL; tok->type_comments = 0; - tok->interactive_underflow = IUNDERFLOW_NORMAL; - tok->str = NULL; - tok->report_warnings = 1; tok->tok_extra_tokens = 0; tok->comment_newline = 0; tok->implicit_newline = 0; @@ -100,13 +87,12 @@ _PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, const { token->level = tok->level; token->span = _PyLexer_BufferSpan(tok, start, end); - int lineno = ISSTRINGLIT(type) ? tok->first_lineno : tok->lineno; - token->start_loc = (_PyTok_Loc){lineno, -1}; - token->end_loc = (_PyTok_Loc){tok->lineno, -1}; - if (start != NULL && end != NULL) { - token->start_loc.byte_col = tok->starting_col_offset; - token->end_loc.byte_col = tok->col_offset; + token->start_loc = tok->start_loc; + token->end_loc = (_PyTok_Loc){tok->lineno, _PyLexer_ByteColumn(tok)}; + } + else { + token->start_loc = token->end_loc = (_PyTok_Loc){tok->lineno, -1}; } return type; } diff --git a/Parser/lexer/state.h b/Parser/lexer/state.h index c62c681da41275a..f9546ac4183c8cc 100644 --- a/Parser/lexer/state.h +++ b/Parser/lexer/state.h @@ -13,14 +13,6 @@ #define INSIDE_FSTRING_EXPR_AT_TOP(tok) \ (tok->curly_bracket_depth - tok->curly_bracket_expr_start_depth == 1) -enum interactive_underflow_t { - /* Normal mode of operation: return a new token when asked in interactive mode */ - IUNDERFLOW_NORMAL, - /* Forcefully return ENDMARKER when asked for a new token in interactive mode. This - * can be used to prevent the tokenizer to prompt the user for new tokens */ - IUNDERFLOW_STOP, -}; - struct token { int level; _PyTok_Span span; @@ -51,8 +43,7 @@ typedef struct _tokenizer_mode { int quote_size; int raw; _PyTok_Off start; - _PyTok_Off multi_line_start; - int first_line; + _PyTok_Loc start_loc; _PyTok_Span expr_span; int in_debug; @@ -77,24 +68,16 @@ struct tok_state { char *cur; /* Next character in buffer */ char *inp; /* End of data in buffer */ _PyTok_Off buf_offset; /* Logical offset of buf[0]. */ - int fp_interactive; /* If the file descriptor is interactive */ - char *interactive_src_start; /* The start of the source parsed so far in interactive mode */ - char *interactive_src_end; /* The end of the source parsed so far in interactive mode */ const char *start; /* Start of current token if not NULL */ int done; /* E_OK normally, E_EOF at EOF, otherwise error code */ /* NB If done != E_OK, cur must be == inp!!! */ FILE *fp; /* Rest of input; NULL if tokenizing a string */ - int tabsize; /* Tab spacing */ int indent; /* Current indentation index */ int indstack[MAXINDENT]; /* Stack of indents */ int atbol; /* Nonzero if at begin of new line */ int pendin; /* Pending indents (if > 0) or dedents (if < 0) */ - const char *prompt; /* For interactive prompting */ int lineno; /* Current line number */ - int first_lineno; /* First line of a single line or multi line string - expression (cf. issue 16806) */ - int starting_col_offset; /* The column offset at the beginning of a token */ - int col_offset; /* Current col offset */ + _PyTok_Loc start_loc; int level; /* () [] {} Parentheses nesting level */ /* Used to allow free continuations inside them */ char parenstack[MAXLEVEL]; @@ -105,23 +88,13 @@ struct tok_state { /* Stuff for checking on different tab sizes */ int altindstack[MAXINDENT]; /* Stack of alternate indents */ /* Stuff for PEP 0263 */ - int input_error; char *encoding; /* Source encoding. */ const char* line_start; /* pointer to start of current line */ - const char* multi_line_start; /* pointer to start of first line of - a single line or multi line string - expression (cf. issue 16806) */ - char* str; /* Source string being tokenized (if tokenizing from a string)*/ - _PyTok_SourceText source; struct _PyTok_Reader *reader; int type_comments; /* Whether to look for type comments */ - /* How to proceed when asked for a new token in interactive mode */ - enum interactive_underflow_t interactive_underflow; - int report_warnings; - // TODO: Factor this into its own thing tokenizer_mode tok_mode_stack[MAXFSTRINGLEVEL]; int tok_mode_stack_index; tokenizer_comments *ftstring_comments; @@ -144,6 +117,16 @@ _PyLexer_BufferOffset(const struct tok_state *tok, const char *position) return tok->buf_offset + offset; } +static inline int +_PyLexer_ByteColumn(const struct tok_state *tok) +{ + assert(tok->line_start != NULL); + assert(tok->cur >= tok->line_start); + Py_ssize_t column = tok->cur - tok->line_start; + assert(column <= INT_MAX); + return (int)column; +} + static inline _PyTok_Span _PyLexer_BufferSpan(const struct tok_state *tok, const char *start, const char *end) diff --git a/Parser/lexer/string.c b/Parser/lexer/string.c index 6e498696dd5ff7c..989951abce558ba 100644 --- a/Parser/lexer/string.c +++ b/Parser/lexer/string.c @@ -31,6 +31,15 @@ span_view(const struct tok_state *tok, _PyTok_Span span, Py_ssize_t *length) return _PyTok_SourceSpanView(&tok->source, span, length); } +static void +rewind_to_string_start(struct tok_state *tok, const char *start, + _PyTok_Loc location) +{ + tok->cur = (char *)start + 1; + tok->line_start = start - location.byte_col; + tok->lineno = location.lineno; +} + static tokenizer_comments * current_comments(const struct tok_state *tok) { @@ -226,13 +235,6 @@ _PyLexer_scan_fstring_start(struct tok_state *tok, struct token *token, int c) int quote = c; int quote_size = 1; /* 1 or 3 */ - /* Nodes of type STRING, especially multi line strings - must be handled differently in order to get both - the starting line number and the column offset right. - (cf. issue 16806) */ - tok->first_lineno = tok->lineno; - tok->multi_line_start = tok->line_start; - /* Find the quote size and start of string */ int after_quote = tok_nextc(tok); if (after_quote == quote) { @@ -241,7 +243,6 @@ _PyLexer_scan_fstring_start(struct tok_state *tok, struct token *token, int c) quote_size = 3; } else { - // TODO: Check this tok_backup(tok, after_after_quote); tok_backup(tok, after_quote); } @@ -261,9 +262,7 @@ _PyLexer_scan_fstring_start(struct tok_state *tok, struct token *token, int c) the_current_tok->quote = quote; the_current_tok->quote_size = quote_size; the_current_tok->start = _PyLexer_BufferOffset(tok, tok->start); - the_current_tok->multi_line_start = - _PyLexer_BufferOffset(tok, tok->line_start); - the_current_tok->first_line = tok->lineno; + the_current_tok->start_loc = tok->start_loc; the_current_tok->expr_span = (_PyTok_Span){-1, -1}; the_current_tok->in_format_spec = 0; the_current_tok->in_debug = 0; @@ -307,13 +306,6 @@ _PyLexer_scan_string(struct tok_state *tok, struct token *token, int c) int end_quote_size = 0; int has_escaped_quote = 0; - /* Nodes of type STRING, especially multi line strings - must be handled differently in order to get both - the starting line number and the column offset right. - (cf. issue 16806) */ - tok->first_lineno = tok->lineno; - tok->multi_line_start = tok->line_start; - /* Find the quote size and start of string */ c = tok_nextc(tok); if (c == quote) { @@ -339,15 +331,8 @@ _PyLexer_scan_string(struct tok_state *tok, struct token *token, int c) break; } if (c == EOF || (quote_size == 1 && c == '\n')) { - assert(tok->multi_line_start != NULL); - // shift the tok_state's location into - // the start of string, and report the error - // from the initial quote character - tok->cur = (char *)tok->start; - tok->cur++; - tok->line_start = tok->multi_line_start; - int start = tok->lineno; - tok->lineno = tok->first_lineno; + int end_lineno = tok->lineno; + rewind_to_string_start(tok, tok->start, tok->start_loc); if (INSIDE_FSTRING(tok)) { /* When we are in an f-string, before raising the @@ -365,7 +350,7 @@ _PyLexer_scan_string(struct tok_state *tok, struct token *token, int c) if (quote_size == 3) { _PyTokenizer_syntaxerror(tok, "unterminated triple-quoted string literal" - " (detected at line %d)", start); + " (detected at line %d)", end_lineno); if (c != '\n') { tok->done = E_EOFS; } @@ -377,11 +362,11 @@ _PyLexer_scan_string(struct tok_state *tok, struct token *token, int c) tok, "unterminated string literal (detected at line %d); " "perhaps you escaped the end quote?", - start + end_lineno ); } else { _PyTokenizer_syntaxerror( - tok, "unterminated string literal (detected at line %d)", start + tok, "unterminated string literal (detected at line %d)", end_lineno ); } if (c != '\n') { @@ -421,8 +406,7 @@ _PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, st int unicode_escape = 0; tok->start = tok->cur; - tok->first_lineno = tok->lineno; - tok->starting_col_offset = tok->col_offset; + tok->start_loc = (_PyTok_Loc){tok->lineno, _PyLexer_ByteColumn(tok)}; // If we start with a bracket, we defer to the normal mode as there is nothing for us to tokenize // before it. @@ -466,9 +450,6 @@ _PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, st f_string_middle: - // TODO: This is a bit of a hack, but it works for now. We need to find a better way to handle - // this. - tok->multi_line_start = tok->line_start; while (end_quote_size != current_tok->quote_size) { int c = tok_nextc(tok); if (tok->done == E_ERROR || tok->done == E_DECODE) { @@ -481,7 +462,7 @@ _PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, st ); if (c == EOF || (current_tok->quote_size == 1 && c == '\n')) { - if (tok->input_error) { + if (tok_failed(tok)) { return MAKE_TOKEN(ERRORTOKEN); } @@ -506,23 +487,16 @@ _PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, st return MAKE_TOKEN(FTSTRING_MIDDLE(current_tok)); } - assert(tok->multi_line_start != NULL); - // shift the tok_state's location into - // the start of string, and report the error - // from the initial quote character - tok->cur = offset_pointer(tok, current_tok->start) + 1; - tok->line_start = offset_pointer( - tok, current_tok->multi_line_start); - int start = tok->lineno; - - tokenizer_mode *the_current_tok = TOK_GET_MODE(tok); - tok->lineno = the_current_tok->first_line; + int end_lineno = tok->lineno; + rewind_to_string_start(tok, + offset_pointer(tok, current_tok->start), + current_tok->start_loc); if (current_tok->quote_size == 3) { _PyTokenizer_syntaxerror(tok, "unterminated triple-quoted %c-string literal" " (detected at line %d)", - TOK_GET_STRING_PREFIX(tok), start); + TOK_GET_STRING_PREFIX(tok), end_lineno); if (c != '\n') { tok->done = E_EOFS; } @@ -531,7 +505,7 @@ _PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, st else { return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "unterminated %c-string literal (detected at" - " line %d)", TOK_GET_STRING_PREFIX(tok), start)); + " line %d)", TOK_GET_STRING_PREFIX(tok), end_lineno)); } } diff --git a/Parser/pegen.c b/Parser/pegen.c index d86dd22444e6a7b..49d2355793f0d7d 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -8,8 +8,9 @@ #include #include "lexer/lexer.h" -#include "tokenizer/tokenizer.h" #include "tokenizer/helpers.h" +#include "tokenizer/reader.h" +#include "tokenizer/tokenizer.h" #include "pegen.h" #define IDENTIFIER_CACHE_SIZE 2048 // Must be a power of two. @@ -943,9 +944,7 @@ reset_parser_state_for_error_pass(Parser *p) } p->mark = 0; p->call_invalid_rules = 1; - // Don't try to get extra tokens in interactive mode when trying to - // raise specialized errors in the second pass. - p->tok->interactive_underflow = IUNDERFLOW_STOP; + _PyTok_ReaderStopInteractive(p->tok); } static inline int @@ -961,12 +960,9 @@ _PyPegen_set_syntax_error_metadata(Parser *p) { PyErr_SetRaisedException(exc); return; } - const char *source = NULL; - if (p->tok->str != NULL) { - source = p->tok->str; - } - if (!source && p->tok->fp_interactive && p->tok->interactive_src_start) { - source = p->tok->interactive_src_start; + const char *source = p->tok->source.bytes; + if (source == NULL && p->tok->fp == NULL) { + source = _PyTok_SourceData(&p->tok->source); } PyObject* the_source = NULL; if (source) { @@ -1070,10 +1066,6 @@ _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filena } return NULL; } - if (!tok->fp || ps1 != NULL || ps2 != NULL || - PyUnicode_CompareWithASCIIString(filename_ob, "") == 0) { - tok->fp_interactive = 1; - } // This transfers the ownership to the tokenizer tok->filename = Py_NewRef(filename_ob); @@ -1095,8 +1087,8 @@ _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filena result = _PyPegen_run_parser(p); _PyPegen_Parser_Free(p); - if (tok->fp_interactive && tok->interactive_src_start && result && interactive_src != NULL) { - *interactive_src = PyUnicode_FromString(tok->interactive_src_start); + if (tok->source.bytes != NULL && result && interactive_src != NULL) { + *interactive_src = PyUnicode_FromString(tok->source.bytes); if (!*interactive_src || _PyArena_AddPyObject(arena, *interactive_src) < 0) { Py_XDECREF(*interactive_src); result = NULL; diff --git a/Parser/pegen_errors.c b/Parser/pegen_errors.c index b13e1c079220a92..72c22cea77cddf3 100644 --- a/Parser/pegen_errors.c +++ b/Parser/pegen_errors.c @@ -7,6 +7,7 @@ #include "lexer/state.h" #include "lexer/lexer.h" #include "pegen.h" +#include "tokenizer/reader.h" // TOKENIZER ERRORS @@ -122,7 +123,7 @@ _PyPegen_tokenize_full_source_to_check_for_errors(Parser *p) { // before the one that we had for the generic error. // We don't want to tokenize to the end for interactive input - if (p->tok->prompt != NULL) { + if (_PyTok_ReaderIsInteractive(p->tok)) { return 0; } @@ -225,45 +226,23 @@ _PyPegen_raise_error(Parser *p, PyObject *errtype, int use_mark, const char *err } static PyObject * -get_error_line_from_tokenizer_buffers(Parser *p, Py_ssize_t lineno) +get_error_line_from_source(Parser *p, Py_ssize_t lineno) { - /* If the file descriptor is interactive, the source lines of the current - * (multi-line) statement are stored in p->tok->interactive_src_start. - * If not, we're parsing from a string, which means that the whole source - * is stored in p->tok->str. */ - assert((p->tok->fp == NULL && p->tok->str != NULL) || p->tok->fp != NULL); - - char *cur_line = p->tok->fp_interactive ? p->tok->interactive_src_start : p->tok->str; - if (cur_line == NULL) { - assert(p->tok->fp_interactive); - // We can reach this point if the tokenizer buffers for interactive source have not been - // initialized because we failed to decode the original source with the given locale. - return Py_GetConstant(Py_CONSTANT_EMPTY_STR); - } + const char *cur_line = _PyTok_SourceData(&p->tok->source); Py_ssize_t relative_lineno = p->starting_lineno ? lineno - p->starting_lineno + 1 : lineno; - const char* buf_end = p->tok->fp_interactive ? p->tok->interactive_src_end : p->tok->inp; - - if (buf_end < cur_line) { - buf_end = cur_line + strlen(cur_line); - } + const char *buf_end = cur_line + p->tok->source.len; for (int i = 0; i < relative_lineno - 1; i++) { - char *new_line = strchr(cur_line, '\n'); - // The assert is here for debug builds but the conditional that - // follows is there so in release builds we do not crash at the cost - // to report a potentially wrong line. - assert(new_line != NULL && new_line + 1 < buf_end); - if (new_line == NULL || new_line + 1 > buf_end) { + const char *new_line = memchr(cur_line, '\n', buf_end - cur_line); + if (new_line == NULL) { break; } cur_line = new_line + 1; } - char *next_newline; - if ((next_newline = strchr(cur_line, '\n')) == NULL) { // This is the last line - next_newline = cur_line + strlen(cur_line); - } + const char *next_newline = memchr(cur_line, '\n', buf_end - cur_line); + next_newline = next_newline != NULL ? next_newline : buf_end; return PyUnicode_DecodeUTF8(cur_line, next_newline - cur_line, "replace"); } @@ -295,8 +274,9 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype, goto error; } - if (p->tok->fp_interactive && p->tok->interactive_src_start != NULL) { - error_line = get_error_line_from_tokenizer_buffers(p, lineno); + if (_PyTok_ReaderIsInteractive(p->tok) && + p->tok->source.bytes != NULL) { + error_line = get_error_line_from_source(p, lineno); } else if (p->start_rule == Py_file_input) { error_line = _PyErr_ProgramDecodedTextObject(p->tok->filename, @@ -318,7 +298,7 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype, error_line = PyUnicode_DecodeUTF8(p->tok->line_start, size, "replace"); } else if (p->tok->fp == NULL || p->tok->fp == stdin) { - error_line = get_error_line_from_tokenizer_buffers(p, lineno); + error_line = get_error_line_from_source(p, lineno); } else { error_line = Py_GetConstant(Py_CONSTANT_EMPTY_STR); diff --git a/Parser/tokenizer/decoder.c b/Parser/tokenizer/decoder.c index af17b8b63235f51..5564b69645b500d 100644 --- a/Parser/tokenizer/decoder.c +++ b/Parser/tokenizer/decoder.c @@ -399,10 +399,9 @@ _PyTok_PrepareString(struct tok_state *tok, const char *input, int utf8_only, if (stored < 0) { return -1; } - tok->str = tok->source.bytes != NULL ? tok->source.bytes : (char *)""; if (!utf8_only && (tok->encoding == NULL || strcmp(tok->encoding, "utf-8") == 0) && - !_PyTokenizer_ensure_utf8(tok->str, tok, 1)) { + !_PyTokenizer_ensure_utf8(_PyTok_SourceData(&tok->source), tok, 1)) { return -1; } return 0; diff --git a/Parser/tokenizer/helpers.c b/Parser/tokenizer/helpers.c index bbd64760a18a664..eded08a388ed70e 100644 --- a/Parser/tokenizer/helpers.c +++ b/Parser/tokenizer/helpers.c @@ -98,10 +98,6 @@ _PyTokenizer_indenterror(struct tok_state *tok) int _PyTokenizer_warn_invalid_escape_sequence(struct tok_state *tok, int first_invalid_escape_char) { - if (!tok->report_warnings) { - return 0; - } - PyObject *msg = PyUnicode_FromFormat( "\"\\%c\" is an invalid escape sequence. " "Such sequences will not work in the future. " @@ -187,10 +183,6 @@ _PyTokenizer_raise_init_error(PyObject *filename) int _PyTokenizer_parser_warn(struct tok_state *tok, PyObject *category, const char *format, ...) { - if (!tok->report_warnings) { - return 0; - } - PyObject *errmsg; va_list vargs; va_start(vargs, format); diff --git a/Parser/tokenizer/helpers.h b/Parser/tokenizer/helpers.h index 5edf5a3dfd2e0d1..51de0cbb156f833 100644 --- a/Parser/tokenizer/helpers.h +++ b/Parser/tokenizer/helpers.h @@ -5,10 +5,6 @@ #include "../lexer/state.h" -#define ADVANCE_LINENO() \ - tok->lineno++; \ - tok->col_offset = 0; - int _PyTokenizer_syntaxerror(struct tok_state *tok, const char *format, ...); int _PyTokenizer_syntaxerror_known_range(struct tok_state *tok, int col_offset, int end_col_offset, const char *format, ...); int _PyTokenizer_indenterror(struct tok_state *tok); diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c index c6602286e8248e8..e312f47fa1bc3f7 100644 --- a/Parser/tokenizer/reader.c +++ b/Parser/tokenizer/reader.c @@ -5,8 +5,6 @@ #include "helpers.h" #include "reader.h" #include "reader_internal.h" -#include "../lexer/buffer.h" -#include "../lexer/lexer.h" #include "../lexer/state.h" #ifdef HAVE_UNISTD_H @@ -19,6 +17,39 @@ reader_is_streaming(_PyTok_ReaderKind kind) return kind == _PYTOK_READER_FILE || kind == _PYTOK_READER_READLINE; } +typedef struct { + Py_ssize_t buf; + Py_ssize_t cur; + Py_ssize_t inp; + Py_ssize_t start; + Py_ssize_t line_start; +} BufferOffsets; + +static BufferOffsets +save_buffer_offsets(const struct tok_state *tok, const char *base) +{ + return (BufferOffsets) { + .buf = tok->buf - base, + .cur = tok->cur - base, + .inp = tok->inp - base, + .start = tok->start == NULL ? -1 : tok->start - base, + .line_start = tok->line_start == NULL + ? -1 : tok->line_start - base, + }; +} + +static void +restore_buffer_offsets(struct tok_state *tok, char *base, + const BufferOffsets *offsets) +{ + tok->buf = base + offsets->buf; + tok->cur = base + offsets->cur; + tok->inp = base + offsets->inp; + tok->start = offsets->start < 0 ? NULL : base + offsets->start; + tok->line_start = offsets->line_start < 0 + ? NULL : base + offsets->line_start; +} + void _PyTok_ReaderFree(struct tok_state *tok) { @@ -76,13 +107,12 @@ reserve_input_buffer(struct tok_state *tok, Py_ssize_t needed) assert(tok->buf != NULL); assert(tok->cur >= tok->buf && tok->cur <= tok->inp); assert(tok->inp - tok->buf <= reader->input_buffer_cap); - _PyLexer_BufferPointers pointers; - _PyLexer_SaveBufferPointers(tok, tok->buf, &pointers); + BufferOffsets offsets = save_buffer_offsets(tok, tok->buf); if (reserve_buffer( &tok->buf, &reader->input_buffer_cap, needed) < 0) { return -1; } - _PyLexer_RestoreBufferPointers(tok, tok->buf, &pointers); + restore_buffer_offsets(tok, tok->buf, &offsets); return 0; } @@ -498,13 +528,13 @@ static _PyTok_ReadResult next_interactive(struct tok_state *tok, _PyTok_Chunk *chunk) { _PyTok_Reader *reader = tok->reader; - if (tok->interactive_underflow == IUNDERFLOW_STOP) { + if (reader->stop_interactive) { return _PYTOK_READ_STOPPED; } char *input = PyOS_Readline( - tok->fp != NULL ? tok->fp : stdin, stdout, tok->prompt); + tok->fp != NULL ? tok->fp : stdin, stdout, reader->prompt); if (reader->nextprompt != NULL) { - tok->prompt = reader->nextprompt; + reader->prompt = reader->nextprompt; } if (input == NULL) { return _PYTOK_READ_INTERRUPT; @@ -538,6 +568,20 @@ next_interactive(struct tok_state *tok, _PyTok_Chunk *chunk) return _PYTOK_READ_LINE; } +int +_PyTok_ReaderIsInteractive(const struct tok_state *tok) +{ + return tok->reader->kind == _PYTOK_READER_INTERACTIVE; +} + +void +_PyTok_ReaderStopInteractive(struct tok_state *tok) +{ + if (_PyTok_ReaderIsInteractive(tok)) { + tok->reader->stop_interactive = 1; + } +} + static _PyTok_ReadResult reader_next(struct tok_state *tok, _PyTok_Chunk *chunk) { @@ -564,6 +608,7 @@ reset_streaming_buffer(struct tok_state *tok) assert(tok->buf_offset <= PY_SSIZE_T_MAX - consumed); tok->buf_offset += consumed; tok->cur = tok->inp = tok->buf; + tok->line_start = tok->buf; } int @@ -590,7 +635,6 @@ _PyTok_ReaderUnderflow(struct tok_state *tok) tok->done = E_INTR; } else { - tok->input_error = 1; if (tok->done == E_OK) { tok->done = PyErr_ExceptionMatches(PyExc_MemoryError) ? E_NOMEM : E_ERROR; @@ -621,7 +665,6 @@ _PyTok_ReaderUnderflow(struct tok_state *tok) if (overflow || reserve_input_buffer(tok, used + scan_len + 1) < 0) { _PyTok_ChunkClear(&chunk); tok->done = E_NOMEM; - tok->input_error = 1; return 0; } memcpy(tok->inp, chunk.data, (size_t)scan_len); @@ -629,12 +672,9 @@ _PyTok_ReaderUnderflow(struct tok_state *tok) *tok->inp = '\0'; } else if (!prepared) { - int source_will_grow = - chunk.len > tok->source.cap - tok->source.len - 1; - _PyLexer_BufferPointers pointers; - if (!reset_buffer && source_will_grow) { - _PyLexer_SaveBufferPointers( - tok, tok->source.bytes, &pointers); + BufferOffsets offsets; + if (!reset_buffer) { + offsets = save_buffer_offsets(tok, tok->source.bytes); } _PyTok_Off source_start = _PyTok_SourceAppendLine( &tok->source, chunk.data, chunk.len, @@ -643,7 +683,6 @@ _PyTok_ReaderUnderflow(struct tok_state *tok) _PyTok_ChunkClear(&chunk); tok->done = PyErr_ExceptionMatches(PyExc_MemoryError) ? E_NOMEM : E_ERROR; - tok->input_error = 1; return 0; } if (reset_buffer) { @@ -651,18 +690,12 @@ _PyTok_ReaderUnderflow(struct tok_state *tok) tok->buf_offset = source_start; tok->line_start = tok->buf; tok->start = NULL; - tok->multi_line_start = NULL; } - else if (source_will_grow) { - _PyLexer_RestoreBufferPointers( - tok, tok->source.bytes, &pointers); + else { + restore_buffer_offsets(tok, tok->source.bytes, &offsets); } tok->inp = tok->source.bytes + source_start + scan_len; } - if (tok->fp_interactive) { - tok->interactive_src_start = tok->source.bytes; - tok->interactive_src_end = tok->source.bytes + tok->source.len; - } if (prepared) { if (tok->start == NULL) { tok->buf = tok->cur; @@ -672,12 +705,11 @@ _PyTok_ReaderUnderflow(struct tok_state *tok) } tok->implicit_newline = chunk.implicit_newline; - ADVANCE_LINENO(); + tok->lineno++; if (kind == _PYTOK_READER_FILE && (tok->encoding == NULL || strcmp(tok->encoding, "utf-8") == 0) && !_PyTokenizer_ensure_utf8(tok->cur, tok, tok->lineno)) { _PyTok_ChunkClear(&chunk); - tok->input_error = 1; return 0; } _PyTok_ChunkClear(&chunk); @@ -708,6 +740,7 @@ tokenizer_new_with_reader(_PyTok_ReaderKind kind) return NULL; } tok->cur = tok->inp = tok->buf; + tok->line_start = tok->buf; tok->buf[0] = '\0'; } return tok; @@ -726,7 +759,9 @@ tokenizer_from_string(const char *input, int utf8_only, int exec_input, _PyTokenizer_Free(tok); return NULL; } - tok->buf = tok->cur = tok->inp = tok->str; + char *source = (char *)_PyTok_SourceData(&tok->source); + tok->buf = tok->cur = tok->inp = source; + tok->line_start = source; return tok; } @@ -772,7 +807,7 @@ _PyTokenizer_FromFile(FILE *fp, const char *encoding, return NULL; } tok->fp = fp; - tok->prompt = ps1; + tok->reader->prompt = ps1; tok->reader->nextprompt = ps2; return tok; } @@ -826,13 +861,10 @@ _PyTokenizer_FindEncodingFilename(int fd, PyObject *filename) _PyTokenizer_Free(tok); return NULL; } - /* Reporting a warning here could recursively ask for the encoding. */ - tok->report_warnings = 0; - while (tok->lineno < 2 && tok->done == E_OK) { - struct token token; - _PyToken_Init(&token); - _PyTokenizer_Get(tok, &token); - _PyToken_Free(&token); + if (initialize_file(tok) < 0) { + fclose(fp); + _PyTokenizer_Free(tok); + return NULL; } fclose(fp); char *encoding = tok->encoding == NULL diff --git a/Parser/tokenizer/reader.h b/Parser/tokenizer/reader.h index c27bc2aa3fb8197..2913e52b9d563b4 100644 --- a/Parser/tokenizer/reader.h +++ b/Parser/tokenizer/reader.h @@ -5,5 +5,7 @@ struct tok_state; void _PyTok_ReaderFree(struct tok_state *); int _PyTok_ReaderUnderflow(struct tok_state *); +int _PyTok_ReaderIsInteractive(const struct tok_state *); +void _PyTok_ReaderStopInteractive(struct tok_state *); #endif diff --git a/Parser/tokenizer/reader_internal.h b/Parser/tokenizer/reader_internal.h index 49a6f04ec60af27..edb185a2f74223a 100644 --- a/Parser/tokenizer/reader_internal.h +++ b/Parser/tokenizer/reader_internal.h @@ -42,6 +42,7 @@ typedef struct _PyTok_Reader { _PyTok_ReaderKind kind; PyObject *readline; PyObject *decoder; + const char *prompt; const char *nextprompt; Py_ssize_t input_buffer_cap; @@ -61,6 +62,7 @@ typedef struct _PyTok_Reader { int file_initialized; int file_eof; int decoder_finalized; + int stop_interactive; } _PyTok_Reader; struct tok_state; diff --git a/Parser/tokenizer/source.c b/Parser/tokenizer/source.c index e876be6e0026519..6f6601279944fdb 100644 --- a/Parser/tokenizer/source.c +++ b/Parser/tokenizer/source.c @@ -143,7 +143,7 @@ _PyTok_SourceSpanView(const _PyTok_SourceText *source, _PyTok_Span span, return NULL; } *len = span.end - span.start; - return source->bytes == NULL ? "" : source->bytes + span.start; + return _PyTok_SourceData(source) + span.start; } int diff --git a/Parser/tokenizer/source.h b/Parser/tokenizer/source.h index fac17183ceb734e..11521fd682f39fe 100644 --- a/Parser/tokenizer/source.h +++ b/Parser/tokenizer/source.h @@ -27,6 +27,12 @@ typedef struct { Py_ssize_t implicit_cap; } _PyTok_SourceText; +static inline const char * +_PyTok_SourceData(const _PyTok_SourceText *source) +{ + return source->bytes != NULL ? source->bytes : ""; +} + PyAPI_FUNC(void) _PyTok_SourceInit(_PyTok_SourceText *); /* Clear invalidates all spans and views for the source. */ PyAPI_FUNC(void) _PyTok_SourceClear(_PyTok_SourceText *); diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index 71f236b08d93c8f..eb5c0b86a8fbe45 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -287,7 +287,8 @@ tokenizeriter_next(PyObject *op) is_trailing_token = 1; } - const char *line_start = ISSTRINGLIT(type) ? it->tok->multi_line_start : it->tok->line_start; + const char *line_start = ISSTRINGLIT(type) + ? token_start - token.start_loc.byte_col : it->tok->line_start; PyObject* line = NULL; int line_changed = 1; if (it->tok->tok_extra_tokens && is_trailing_token) { diff --git a/Tools/peg_generator/pegen/build.py b/Tools/peg_generator/pegen/build.py index bfd8e43c6912e86..af8027db27234ad 100644 --- a/Tools/peg_generator/pegen/build.py +++ b/Tools/peg_generator/pegen/build.py @@ -128,7 +128,6 @@ def compile_c_extension( str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "number.c"), str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "state.c"), str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "string.c"), - str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "buffer.c"), str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" / "decoder.c"), str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" / "reader.c"), str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" / "helpers.c"),