From f761f85b68a67c52eea485b1a09ac36b16703154 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sat, 11 Jul 2026 15:29:53 +0100 Subject: [PATCH] gh-153569: Split tokenizer lexer into focused files --- Lib/test/test_fstring.py | 10 + Lib/test/test_tokenize.py | 160 ++++ Makefile.pre.in | 3 + PCbuild/_freeze_module.vcxproj | 2 + PCbuild/_freeze_module.vcxproj.filters | 6 + PCbuild/pythoncore.vcxproj | 3 + PCbuild/pythoncore.vcxproj.filters | 9 + Parser/lexer/lexer.c | 974 +------------------------ Parser/lexer/lexer_internal.h | 57 ++ Parser/lexer/number.c | 313 ++++++++ Parser/lexer/string.c | 642 ++++++++++++++++ Tools/peg_generator/pegen/build.py | 2 + 12 files changed, 1224 insertions(+), 957 deletions(-) create mode 100644 Parser/lexer/lexer_internal.h create mode 100644 Parser/lexer/number.c create mode 100644 Parser/lexer/string.c diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index 9a8e56836fc671..45890f8fb35dcb 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -709,6 +709,16 @@ def test_double_braces(self): ["f'{ {{}} }'", # dict in a set ]) + def test_double_brace_ast_location_covers_both_source_braces(self): + value = ast.parse('f"a{{"').body[0].value.values[0] + self.assertIsInstance(value, ast.Constant) + self.assertEqual(value.value, "a{") + self.assertEqual( + (value.lineno, value.col_offset, value.end_lineno, + value.end_col_offset), + (1, 2, 1, 5), + ) + def test_compile_time_concat(self): x = 'def' self.assertEqual('abc' f'## {x}ghi', 'abc## defghi') diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index ab53a20cff5539..e2db09d61f409b 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -2236,6 +2236,13 @@ def test_number_starting_with_zero(self): self.assertEqual(tokens, expected_tokens) class CTokenizeTest(TestCase): + @staticmethod + def _get_tokens(source, *, extra_tokens=False): + return list(tokenize._generate_tokens_from_c_tokenizer( + StringIO(source).readline, + extra_tokens=extra_tokens, + )) + def check_tokenize(self, s, expected): # Format the tokens in s in a table format. # The ENDMARKER and final NEWLINE are omitted. @@ -2266,6 +2273,159 @@ def readline(encoding): )) self.assertEqual(tokens, expected) + def test_extra_tokens_relaxes_lexer_errors(self): + cases = [ + ( + "2sin(x)", + ("invalid decimal literal", (1, 1)), + [ + (token.NUMBER, "2", (1, 0), (1, 1)), + (token.NAME, "sin", (1, 1), (1, 4)), + (token.OP, "(", (1, 4), (1, 5)), + (token.NAME, "x", (1, 5), (1, 6)), + (token.OP, ")", (1, 6), (1, 7)), + ], + ), + ( + "01234", + ( + "leading zeros in decimal integer literals are not permitted; " + "use an 0o prefix for octal integers", + (1, 1), + ), + [(token.NUMBER, "01234", (1, 0), (1, 5))], + ), + ( + ")", + ("unmatched ')'", (1, 1)), + [(token.OP, ")", (1, 0), (1, 1))], + ), + ( + "(]", + ( + "closing parenthesis ']' does not match opening parenthesis '('", + (1, 2), + ), + [ + (token.OP, "(", (1, 0), (1, 1)), + (token.OP, "]", (1, 1), (1, 2)), + ], + ), + ( + "a☃b", + ("invalid character '☃' (U+2603)", (1, 2)), + [(token.NAME, "a☃b", (1, 0), (1, 3))], + ), + ] + + for source, error, expected in cases: + with self.subTest(source=source): + with self.assertRaises(tokenize.TokenError) as caught: + self._get_tokens(source) + self.assertEqual(caught.exception.args, error) + + tokens = self._get_tokens(source, extra_tokens=True) + self.assertEqual( + [(tok.type, tok.string, tok.start, tok.end) + for tok in tokens[:-2]], + expected, + ) + + def test_extra_tokens_emits_comments_and_real_indent_locations(self): + source = "if x:\n # c\n y\nz\n" + kinds = {token.COMMENT, token.NL, token.INDENT, token.DEDENT} + + parser_tokens = self._get_tokens(source) + self.assertEqual( + [(tok.type, tok.string, tok.start, tok.end) + for tok in parser_tokens if tok.type in kinds], + [ + (token.INDENT, "", (3, -1), (3, -1)), + (token.DEDENT, "", (4, -1), (4, -1)), + ], + ) + + tolerant_tokens = self._get_tokens(source, extra_tokens=True) + self.assertEqual( + [(tok.type, tok.string, tok.start, tok.end) + for tok in tolerant_tokens if tok.type in kinds], + [ + (token.COMMENT, "# c", (2, 2), (2, 5)), + (token.NL, "\n", (2, 5), (2, 6)), + (token.INDENT, " ", (3, 0), (3, 2)), + (token.DEDENT, "", (4, 0), (4, 0)), + ], + ) + + def test_printable_invalid_operator_streams_in_both_modes(self): + for extra_tokens in (False, True): + with self.subTest(extra_tokens=extra_tokens): + first = self._get_tokens( + "1 $ 2", + extra_tokens=extra_tokens, + )[1] + self.assertEqual( + (first.type, first.string, first.start, first.end), + (token.OP, "$", (1, 2), (1, 3)), + ) + + def test_degraded_fstring_format_spec(self): + tokens = self._get_tokens('f"{1:{2}{{3}}}"') + self.assertEqual( + [(tok.string, tok.start, tok.end) + for tok in tokens if tok.type == token.FSTRING_MIDDLE], + [ + ("{", (1, 8), (1, 9)), + ("3", (1, 10), (1, 11)), + ("}", (1, 12), (1, 13)), + ], + ) + + tokens = self._get_tokens('f"{x:{y}}"') + self.assertEqual( + [(tok.string, tok.start, tok.end) + for tok in tokens if tok.type == token.FSTRING_MIDDLE], + [("", (1, 8), (1, 8))], + ) + + with self.assertRaises(tokenize.TokenError) as caught: + self._get_tokens('f"{1:{2}x}}y"') + self.assertEqual( + caught.exception.args, + ("f-string: single '}' is not allowed", (1, 11)), + ) + + def test_escaped_fstring_brace_has_a_position_gap(self): + tokens = self._get_tokens('f"a{{"', extra_tokens=True) + self.assertEqual( + [(tok.type, tok.string, tok.start, tok.end) + for tok in tokens + if tok.type in {token.FSTRING_MIDDLE, token.FSTRING_END}], + [ + (token.FSTRING_MIDDLE, "a{", (1, 2), (1, 4)), + (token.FSTRING_END, '"', (1, 5), (1, 6)), + ], + ) + + def test_tolerant_incompatible_prefix_position_after_non_ascii(self): + with self.assertRaises(tokenize.TokenError) as caught: + self._get_tokens('bé )tf"2 ', extra_tokens=True) + self.assertEqual( + caught.exception.args, + ("'f' and 't' prefixes are incompatible", (1, 6)), + ) + + def test_tolerant_fstring_closer_at_expression_entry_depth(self): + source = "f'1:{]]{}}r}''" + for extra_tokens, position in ((False, (1, 6)), (True, (1, 7))): + with self.subTest(extra_tokens=extra_tokens): + with self.assertRaises(tokenize.TokenError) as caught: + self._get_tokens(source, extra_tokens=extra_tokens) + self.assertEqual( + caught.exception.args, + ("f-string: unmatched ']'", position), + ) + def test_int(self): self.check_tokenize('0xff <= 255', """\ diff --git a/Makefile.pre.in b/Makefile.pre.in index 786f4fd5c00247..70b8702c808152 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -395,7 +395,9 @@ PEGEN_OBJS= \ TOKENIZER_OBJS= \ Parser/lexer/buffer.o \ Parser/lexer/lexer.o \ + Parser/lexer/number.o \ Parser/lexer/state.o \ + Parser/lexer/string.o \ Parser/tokenizer/file_tokenizer.o \ Parser/tokenizer/readline_tokenizer.o \ Parser/tokenizer/string_tokenizer.o \ @@ -410,6 +412,7 @@ PEGEN_HEADERS= \ TOKENIZER_HEADERS= \ Parser/lexer/buffer.h \ Parser/lexer/lexer.h \ + Parser/lexer/lexer_internal.h \ Parser/lexer/state.h \ Parser/tokenizer/tokenizer.h \ Parser/tokenizer/helpers.h diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 5a47a263516ae0..04bd36404366a5 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -184,6 +184,8 @@ + + diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 83a2d60ec847b3..7710acf43b5137 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -463,6 +463,12 @@ Source Files + + Source Files + + + Source Files + Source Files diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index e255ed5af19125..47296a93424a8f 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -420,6 +420,7 @@ + @@ -584,6 +585,8 @@ + + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 649ee1859ff996..124f6c7bb90de6 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -327,6 +327,9 @@ Parser + + Parser + Parser @@ -1331,6 +1334,12 @@ Parser + + Parser + + + Parser + Parser diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index 7f25afec302c22..41f03a408cfc75 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -3,44 +3,13 @@ #include "pycore_unicodeobject.h" #include "errcode.h" -#include "state.h" +#include "lexer_internal.h" #include "../tokenizer/helpers.h" /* Alternate tab spacing */ #define ALTTABSIZE 1 -#define is_potential_identifier_start(c) (\ - (c >= 'a' && c <= 'z')\ - || (c >= 'A' && c <= 'Z')\ - || c == '_'\ - || (c >= 128)) -#define is_potential_identifier_char(c) (\ - (c >= 'a' && c <= 'z')\ - || (c >= 'A' && c <= 'Z')\ - || (c >= '0' && c <= '9')\ - || c == '_'\ - || (c >= 128)) - -#ifdef Py_DEBUG -static inline tokenizer_mode* TOK_GET_MODE(struct tok_state* tok) { - assert(tok->tok_mode_stack_index >= 0); - assert(tok->tok_mode_stack_index < MAXFSTRINGLEVEL); - return &(tok->tok_mode_stack[tok->tok_mode_stack_index]); -} -static inline tokenizer_mode* TOK_NEXT_MODE(struct tok_state* tok) { - assert(tok->tok_mode_stack_index >= 0); - assert(tok->tok_mode_stack_index + 1 < MAXFSTRINGLEVEL); - return &(tok->tok_mode_stack[++tok->tok_mode_stack_index]); -} -#else -#define TOK_GET_MODE(tok) (&(tok->tok_mode_stack[tok->tok_mode_stack_index])) -#define TOK_NEXT_MODE(tok) (&(tok->tok_mode_stack[++tok->tok_mode_stack_index])) -#endif - -#define FTSTRING_MIDDLE(tok_mode) (tok_mode->string_kind == TSTRING ? TSTRING_MIDDLE : FSTRING_MIDDLE) -#define FTSTRING_END(tok_mode) (tok_mode->string_kind == TSTRING ? TSTRING_END : FSTRING_END) -#define TOK_GET_STRING_PREFIX(tok) (TOK_GET_MODE(tok)->string_kind == TSTRING ? 't' : 'f') #define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end) #define MAKE_TYPE_COMMENT_TOKEN(token_type, col_offset, end_col_offset) (\ _PyLexer_type_comment_token_setup(tok, token, token_type, col_offset, end_col_offset, p_start, p_end)) @@ -56,8 +25,8 @@ contains_null_bytes(const char* str, size_t size) } /* Get next char, updating state; error code goes into tok->done */ -static int -tok_nextc(struct tok_state *tok) +int +_PyLexer_nextc(struct tok_state *tok) { int rc; for (;;) { @@ -96,8 +65,8 @@ tok_nextc(struct tok_state *tok) } /* Back-up one character */ -static void -tok_backup(struct tok_state *tok, int c) +void +_PyLexer_backup(struct tok_state *tok, int c) { if (c != EOF) { if (--tok->cur < tok->buf) { @@ -110,254 +79,7 @@ tok_backup(struct tok_state *tok, int c) } } -static int -set_ftstring_expr(struct tok_state* tok, struct token *token, char c) { - assert(token != NULL); - assert(c == '}' || c == ':' || c == '!'); - tokenizer_mode *tok_mode = TOK_GET_MODE(tok); - - if (!(tok_mode->in_debug || tok_mode->string_kind == TSTRING) || token->metadata) { - return 0; - } - PyObject *res = NULL; - - // Look for a # character outside of string literals - int hash_detected = 0; - int in_string = 0; - char quote_char = 0; - - for (Py_ssize_t i = 0; i < tok_mode->last_expr_size - tok_mode->last_expr_end; i++) { - char ch = tok_mode->last_expr_buffer[i]; - - // Skip escaped characters - if (ch == '\\') { - i++; - continue; - } - - // Handle quotes - if (ch == '"' || ch == '\'') { - // The following if/else block works becase there is an off number - // of quotes in STRING tokens and the lexer only ever reaches this - // function with valid STRING tokens. - // For example: """hello""" - // First quote: in_string = 1 - // Second quote: in_string = 0 - // Third quote: in_string = 1 - if (!in_string) { - in_string = 1; - quote_char = ch; - } - else if (ch == quote_char) { - in_string = 0; - } - continue; - } - - // Check for # outside strings - if (ch == '#' && !in_string) { - hash_detected = 1; - break; - } - } - // If we found a # character in the expression, we need to handle comments - if (hash_detected) { - // Allocate buffer for processed result - char *result = (char *)PyMem_Malloc((tok_mode->last_expr_size - tok_mode->last_expr_end + 1) * sizeof(char)); - if (!result) { - return -1; - } - - Py_ssize_t i = 0; // Input position - Py_ssize_t j = 0; // Output position - in_string = 0; // Whether we're in a string - quote_char = 0; // Current string quote char - - // Process each character - while (i < tok_mode->last_expr_size - tok_mode->last_expr_end) { - char ch = tok_mode->last_expr_buffer[i]; - - // Handle string quotes - if (ch == '"' || ch == '\'') { - // See comment above to understand this part - if (!in_string) { - in_string = 1; - quote_char = ch; - } else if (ch == quote_char) { - in_string = 0; - } - result[j++] = ch; - } - // Skip comments - else if (ch == '#' && !in_string) { - while (i < tok_mode->last_expr_size - tok_mode->last_expr_end && - tok_mode->last_expr_buffer[i] != '\n') { - i++; - } - if (i < tok_mode->last_expr_size - tok_mode->last_expr_end) { - result[j++] = '\n'; - } - } - // Copy other chars - else { - result[j++] = ch; - } - i++; - } - - result[j] = '\0'; // Null-terminate the result string - res = PyUnicode_DecodeUTF8(result, j, NULL); - PyMem_Free(result); - } else { - res = PyUnicode_DecodeUTF8( - tok_mode->last_expr_buffer, - tok_mode->last_expr_size - tok_mode->last_expr_end, - NULL - ); - } - - if (!res) { - return -1; - } - token->metadata = res; - return 0; -} - -int -_PyLexer_update_ftstring_expr(struct tok_state *tok, char cur) -{ - assert(tok->cur != NULL); - - Py_ssize_t size = strlen(tok->cur); - tokenizer_mode *tok_mode = TOK_GET_MODE(tok); - - switch (cur) { - case 0: - if (!tok_mode->last_expr_buffer || tok_mode->last_expr_end >= 0) { - return 1; - } - char *new_buffer = PyMem_Realloc( - tok_mode->last_expr_buffer, - tok_mode->last_expr_size + size - ); - if (new_buffer == NULL) { - PyMem_Free(tok_mode->last_expr_buffer); - goto error; - } - tok_mode->last_expr_buffer = new_buffer; - strncpy(tok_mode->last_expr_buffer + tok_mode->last_expr_size, tok->cur, size); - tok_mode->last_expr_size += size; - break; - case '{': - if (tok_mode->last_expr_buffer != NULL) { - PyMem_Free(tok_mode->last_expr_buffer); - } - tok_mode->last_expr_buffer = PyMem_Malloc(size); - if (tok_mode->last_expr_buffer == NULL) { - goto error; - } - tok_mode->last_expr_size = size; - tok_mode->last_expr_end = -1; - strncpy(tok_mode->last_expr_buffer, tok->cur, size); - break; - case '}': - case '!': - tok_mode->last_expr_end = strlen(tok->start); - break; - case ':': - if (tok_mode->last_expr_end == -1) { - tok_mode->last_expr_end = strlen(tok->start); - } - break; - default: - Py_UNREACHABLE(); - } - return 1; -error: - tok->done = E_NOMEM; - return 0; -} - -static int -lookahead(struct tok_state *tok, const char *test) -{ - const char *s = test; - int res = 0; - while (1) { - int c = tok_nextc(tok); - if (*s == 0) { - res = !is_potential_identifier_char(c); - } - else if (c == *s) { - s++; - continue; - } - - tok_backup(tok, c); - while (s != test) { - tok_backup(tok, *--s); - } - return res; - } -} -static int -verify_end_of_number(struct tok_state *tok, int c, const char *kind) { - if (tok->tok_extra_tokens) { - // When we are parsing extra tokens, we don't want to emit warnings - // about invalid literals, because we want to be a bit more liberal. - return 1; - } - /* Emit a deprecation warning only if the numeric literal is immediately - * followed by one of keywords which can occur after a numeric literal - * in valid code: "and", "else", "for", "if", "in", "is" and "or". - * It allows to gradually deprecate existing valid code without adding - * warning before error in most cases of invalid numeric literal (which - * would be confusing and break existing tests). - * Raise a syntax error with slightly better message than plain - * "invalid syntax" if the numeric literal is immediately followed by - * other keyword or identifier. - */ - int r = 0; - if (c == 'a') { - r = lookahead(tok, "nd"); - } - else if (c == 'e') { - r = lookahead(tok, "lse"); - } - else if (c == 'f') { - r = lookahead(tok, "or"); - } - else if (c == 'i') { - int c2 = tok_nextc(tok); - if (c2 == 'f' || c2 == 'n' || c2 == 's') { - r = 1; - } - tok_backup(tok, c2); - } - else if (c == 'o') { - r = lookahead(tok, "r"); - } - else if (c == 'n') { - r = lookahead(tok, "ot"); - } - if (r) { - tok_backup(tok, c); - if (_PyTokenizer_parser_warn(tok, PyExc_SyntaxWarning, - "invalid %s literal", kind)) - { - return 0; - } - tok_nextc(tok); - } - else /* In future releases, only error will remain. */ - if (c < 128 && is_potential_identifier_char(c)) { - tok_backup(tok, c); - _PyTokenizer_syntaxerror(tok, "invalid %s literal", kind); - return 0; - } - return 1; -} /* Verify that the identifier follows PEP 3131. */ static int @@ -409,27 +131,7 @@ verify_identifier(struct tok_state *tok) return 1; } -static int -tok_decimal_tail(struct tok_state *tok) -{ - int c; - while (1) { - do { - c = tok_nextc(tok); - } while (Py_ISDIGIT(c)); - if (c != '_') { - break; - } - c = tok_nextc(tok); - if (!Py_ISDIGIT(c)) { - tok_backup(tok, c); - _PyTokenizer_syntaxerror(tok, "invalid decimal literal"); - return 0; - } - } - return c; -} static inline int tok_continuation_line(struct tok_state *tok) { @@ -452,53 +154,10 @@ tok_continuation_line(struct tok_state *tok) { return c; } -static int -maybe_raise_syntax_error_for_string_prefixes(struct tok_state *tok, - int saw_b, int saw_r, int saw_u, - int saw_f, int saw_t) { - // Supported: rb, rf, rt (in any order) - // Unsupported: ub, ur, uf, ut, bf, bt, ft (in any order) - -#define RETURN_SYNTAX_ERROR(PREFIX1, PREFIX2) \ - do { \ - (void)_PyTokenizer_syntaxerror_known_range( \ - tok, (int)(tok->start + 1 - tok->line_start), \ - (int)(tok->cur - tok->line_start), \ - "'" PREFIX1 "' and '" PREFIX2 "' prefixes are incompatible"); \ - return -1; \ - } while (0) - - if (saw_u && saw_b) { - RETURN_SYNTAX_ERROR("u", "b"); - } - if (saw_u && saw_r) { - RETURN_SYNTAX_ERROR("u", "r"); - } - if (saw_u && saw_f) { - RETURN_SYNTAX_ERROR("u", "f"); - } - if (saw_u && saw_t) { - RETURN_SYNTAX_ERROR("u", "t"); - } - if (saw_b && saw_f) { - RETURN_SYNTAX_ERROR("b", "f"); - } - if (saw_b && saw_t) { - RETURN_SYNTAX_ERROR("b", "t"); - } - if (saw_f && saw_t) { - RETURN_SYNTAX_ERROR("f", "t"); - } - -#undef RETURN_SYNTAX_ERROR - - return 0; -} - -static int -tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct token *token) +int +_PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct token *token) { int c; int blankline, nonascii; @@ -768,7 +427,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t c = tok_nextc(tok); if (c == '"' || c == '\'') { // Raise error on incompatible string prefixes: - int status = maybe_raise_syntax_error_for_string_prefixes( + int status = _PyLexer_check_string_prefixes( tok, saw_b, saw_r, saw_u, saw_f, saw_t); if (status < 0) { return MAKE_TOKEN(ERRORTOKEN); @@ -776,9 +435,9 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t // Handle valid f or t string creation: if (saw_f || saw_t) { - goto f_string_quote; + return _PyLexer_scan_fstring_start(tok, token, c); } - goto letter_quote; + return _PyLexer_scan_string(tok, token, c); } } while (is_potential_identifier_char(c)) { @@ -832,7 +491,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t if (c == '.') { c = tok_nextc(tok); if (Py_ISDIGIT(c)) { - goto fraction; + return _PyLexer_scan_number(tok, token, c, 1); } else if (c == '.') { c = tok_nextc(tok); if (c == '.') { @@ -853,392 +512,15 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t return MAKE_TOKEN(DOT); } + /* Number */ if (Py_ISDIGIT(c)) { - if (c == '0') { - /* Hex, octal or binary -- maybe. */ - c = tok_nextc(tok); - if (c == 'x' || c == 'X') { - /* Hex */ - c = tok_nextc(tok); - do { - if (c == '_') { - c = tok_nextc(tok); - } - if (!Py_ISXDIGIT(c)) { - tok_backup(tok, c); - return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid hexadecimal literal")); - } - do { - c = tok_nextc(tok); - } while (Py_ISXDIGIT(c)); - } while (c == '_'); - if (!verify_end_of_number(tok, c, "hexadecimal")) { - return MAKE_TOKEN(ERRORTOKEN); - } - } - else if (c == 'o' || c == 'O') { - /* Octal */ - c = tok_nextc(tok); - do { - if (c == '_') { - c = tok_nextc(tok); - } - if (c < '0' || c >= '8') { - if (Py_ISDIGIT(c)) { - return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, - "invalid digit '%c' in octal literal", c)); - } - else { - tok_backup(tok, c); - return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid octal literal")); - } - } - do { - c = tok_nextc(tok); - } while ('0' <= c && c < '8'); - } while (c == '_'); - if (Py_ISDIGIT(c)) { - return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, - "invalid digit '%c' in octal literal", c)); - } - if (!verify_end_of_number(tok, c, "octal")) { - return MAKE_TOKEN(ERRORTOKEN); - } - } - else if (c == 'b' || c == 'B') { - /* Binary */ - c = tok_nextc(tok); - do { - if (c == '_') { - c = tok_nextc(tok); - } - if (c != '0' && c != '1') { - if (Py_ISDIGIT(c)) { - return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid digit '%c' in binary literal", c)); - } - else { - tok_backup(tok, c); - return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid binary literal")); - } - } - do { - c = tok_nextc(tok); - } while (c == '0' || c == '1'); - } while (c == '_'); - if (Py_ISDIGIT(c)) { - return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid digit '%c' in binary literal", c)); - } - if (!verify_end_of_number(tok, c, "binary")) { - return MAKE_TOKEN(ERRORTOKEN); - } - } - else { - int nonzero = 0; - /* maybe old-style octal; c is first char of it */ - /* in any case, allow '0' as a literal */ - while (1) { - if (c == '_') { - c = tok_nextc(tok); - if (!Py_ISDIGIT(c)) { - tok_backup(tok, c); - return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid decimal literal")); - } - } - if (c != '0') { - break; - } - c = tok_nextc(tok); - } - char* zeros_end = tok->cur; - if (Py_ISDIGIT(c)) { - nonzero = 1; - c = tok_decimal_tail(tok); - if (c == 0) { - return MAKE_TOKEN(ERRORTOKEN); - } - } - if (c == '.') { - c = tok_nextc(tok); - goto fraction; - } - else if (c == 'e' || c == 'E') { - goto exponent; - } - else if (c == 'j' || c == 'J') { - goto imaginary; - } - else if (nonzero && !tok->tok_extra_tokens) { - /* Old-style octal: now disallowed. */ - tok_backup(tok, c); - return MAKE_TOKEN(_PyTokenizer_syntaxerror_known_range( - tok, (int)(tok->start + 1 - tok->line_start), - (int)(zeros_end - tok->line_start), - "leading zeros in decimal integer " - "literals are not permitted; " - "use an 0o prefix for octal integers")); - } - if (!verify_end_of_number(tok, c, "decimal")) { - return MAKE_TOKEN(ERRORTOKEN); - } - } - } - else { - /* Decimal */ - c = tok_decimal_tail(tok); - if (c == 0) { - return MAKE_TOKEN(ERRORTOKEN); - } - { - /* Accept floating-point numbers. */ - if (c == '.') { - c = tok_nextc(tok); - fraction: - /* Fraction */ - if (Py_ISDIGIT(c)) { - c = tok_decimal_tail(tok); - if (c == 0) { - return MAKE_TOKEN(ERRORTOKEN); - } - } - } - if (c == 'e' || c == 'E') { - int e; - exponent: - e = c; - /* Exponent part */ - c = tok_nextc(tok); - if (c == '+' || c == '-') { - c = tok_nextc(tok); - if (!Py_ISDIGIT(c)) { - tok_backup(tok, c); - return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid decimal literal")); - } - } else if (!Py_ISDIGIT(c)) { - tok_backup(tok, c); - if (!verify_end_of_number(tok, e, "decimal")) { - return MAKE_TOKEN(ERRORTOKEN); - } - tok_backup(tok, e); - p_start = tok->start; - p_end = tok->cur; - return MAKE_TOKEN(NUMBER); - } - c = tok_decimal_tail(tok); - if (c == 0) { - return MAKE_TOKEN(ERRORTOKEN); - } - } - if (c == 'j' || c == 'J') { - /* Imaginary part */ - imaginary: - c = tok_nextc(tok); - if (!verify_end_of_number(tok, c, "imaginary")) { - return MAKE_TOKEN(ERRORTOKEN); - } - } - else if (!verify_end_of_number(tok, c, "decimal")) { - return MAKE_TOKEN(ERRORTOKEN); - } - } - } - tok_backup(tok, c); - p_start = tok->start; - p_end = tok->cur; - return MAKE_TOKEN(NUMBER); - } - - f_string_quote: - if (((Py_TOLOWER(*tok->start) == 'f' || Py_TOLOWER(*tok->start) == 'r' || Py_TOLOWER(*tok->start) == 't') - && (c == '\'' || c == '"'))) { - - int quote = c; - int quote_size = 1; /* 1 or 3 */ - - /* Nodes of type STRING, especially multi line strings - must be handled differently in order to get both - the starting line number and the column offset right. - (cf. issue 16806) */ - tok->first_lineno = tok->lineno; - tok->multi_line_start = tok->line_start; - - /* Find the quote size and start of string */ - int after_quote = tok_nextc(tok); - if (after_quote == quote) { - int after_after_quote = tok_nextc(tok); - if (after_after_quote == quote) { - quote_size = 3; - } - else { - // TODO: Check this - tok_backup(tok, after_after_quote); - tok_backup(tok, after_quote); - } - } - if (after_quote != quote) { - tok_backup(tok, after_quote); - } - - - p_start = tok->start; - p_end = tok->cur; - if (tok->tok_mode_stack_index + 1 >= MAXFSTRINGLEVEL) { - return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "too many nested f-strings or t-strings")); - } - tokenizer_mode *the_current_tok = TOK_NEXT_MODE(tok); - the_current_tok->kind = TOK_FSTRING_MODE; - the_current_tok->quote = quote; - the_current_tok->quote_size = quote_size; - the_current_tok->start = tok->start; - the_current_tok->multi_line_start = tok->line_start; - the_current_tok->first_line = tok->lineno; - the_current_tok->start_offset = -1; - the_current_tok->multi_line_start_offset = -1; - the_current_tok->last_expr_buffer = NULL; - the_current_tok->last_expr_size = 0; - the_current_tok->last_expr_end = -1; - the_current_tok->in_format_spec = 0; - the_current_tok->in_debug = 0; - - enum string_kind_t string_kind = FSTRING; - switch (*tok->start) { - case 'T': - case 't': - the_current_tok->raw = Py_TOLOWER(*(tok->start + 1)) == 'r'; - string_kind = TSTRING; - break; - case 'F': - case 'f': - the_current_tok->raw = Py_TOLOWER(*(tok->start + 1)) == 'r'; - break; - case 'R': - case 'r': - the_current_tok->raw = 1; - if (Py_TOLOWER(*(tok->start + 1)) == 't') { - string_kind = TSTRING; - } - break; - default: - Py_UNREACHABLE(); - } - - the_current_tok->string_kind = string_kind; - the_current_tok->curly_bracket_depth = 0; - the_current_tok->curly_bracket_expr_start_depth = -1; - return string_kind == TSTRING ? MAKE_TOKEN(TSTRING_START) : MAKE_TOKEN(FSTRING_START); + return _PyLexer_scan_number(tok, token, c, 0); } - letter_quote: /* String */ if (c == '\'' || c == '"') { - int quote = c; - int quote_size = 1; /* 1 or 3 */ - int end_quote_size = 0; - int has_escaped_quote = 0; - - /* Nodes of type STRING, especially multi line strings - must be handled differently in order to get both - the starting line number and the column offset right. - (cf. issue 16806) */ - tok->first_lineno = tok->lineno; - tok->multi_line_start = tok->line_start; - - /* Find the quote size and start of string */ - c = tok_nextc(tok); - if (c == quote) { - c = tok_nextc(tok); - if (c == quote) { - quote_size = 3; - } - else { - end_quote_size = 1; /* empty string found */ - } - } - if (c != quote) { - tok_backup(tok, c); - } - - /* Get rest of string */ - while (end_quote_size != quote_size) { - c = tok_nextc(tok); - if (tok->done == E_ERROR) { - return MAKE_TOKEN(ERRORTOKEN); - } - if (tok->done == E_DECODE) { - break; - } - if (c == EOF || (quote_size == 1 && c == '\n')) { - assert(tok->multi_line_start != NULL); - // shift the tok_state's location into - // the start of string, and report the error - // from the initial quote character - tok->cur = (char *)tok->start; - tok->cur++; - tok->line_start = tok->multi_line_start; - int start = tok->lineno; - tok->lineno = tok->first_lineno; - - if (INSIDE_FSTRING(tok)) { - /* When we are in an f-string, before raising the - * unterminated string literal error, check whether - * does the initial quote matches with f-strings quotes - * and if it is, then this must be a missing '}' token - * so raise the proper error */ - tokenizer_mode *the_current_tok = TOK_GET_MODE(tok); - if (the_current_tok->quote == quote && - the_current_tok->quote_size == quote_size) { - return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, - "%c-string: expecting '}'", TOK_GET_STRING_PREFIX(tok))); - } - } - - if (quote_size == 3) { - _PyTokenizer_syntaxerror(tok, "unterminated triple-quoted string literal" - " (detected at line %d)", start); - if (c != '\n') { - tok->done = E_EOFS; - } - return MAKE_TOKEN(ERRORTOKEN); - } - else { - if (has_escaped_quote) { - _PyTokenizer_syntaxerror( - tok, - "unterminated string literal (detected at line %d); " - "perhaps you escaped the end quote?", - start - ); - } else { - _PyTokenizer_syntaxerror( - tok, "unterminated string literal (detected at line %d)", start - ); - } - if (c != '\n') { - tok->done = E_EOLS; - } - return MAKE_TOKEN(ERRORTOKEN); - } - } - if (c == quote) { - end_quote_size += 1; - } - else { - end_quote_size = 0; - if (c == '\\') { - c = tok_nextc(tok); /* skip escaped char */ - if (c == quote) { /* but record whether the escaped char was a quote */ - has_escaped_quote = 1; - } - if (c == '\r') { - c = tok_nextc(tok); - } - } - } - } - - p_start = tok->start; - p_end = tok->cur; - return MAKE_TOKEN(STRING); + return _PyLexer_scan_string(tok, token, c); } /* Line continuation */ @@ -1264,7 +546,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t if ((cursor_valid) && !_PyLexer_update_ftstring_expr(tok, c)) { return MAKE_TOKEN(ENDMARKER); } - if ((cursor_valid) && c != '{' && set_ftstring_expr(tok, token, c)) { + if ((cursor_valid) && c != '{' && _PyLexer_set_ftstring_expr(tok, token, c)) { return MAKE_TOKEN(ERRORTOKEN); } @@ -1389,237 +671,15 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t return MAKE_TOKEN(_PyToken_OneChar(c)); } -static int -tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct token *token) -{ - const char *p_start = NULL; - const char *p_end = NULL; - int end_quote_size = 0; - int unicode_escape = 0; - - tok->start = tok->cur; - tok->first_lineno = tok->lineno; - tok->starting_col_offset = tok->col_offset; - - // If we start with a bracket, we defer to the normal mode as there is nothing for us to tokenize - // before it. - int start_char = tok_nextc(tok); - if (start_char == '{') { - int peek1 = tok_nextc(tok); - tok_backup(tok, peek1); - tok_backup(tok, start_char); - if (peek1 != '{') { - current_tok->curly_bracket_expr_start_depth++; - if (current_tok->curly_bracket_expr_start_depth >= MAX_EXPR_NESTING) { - return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, - "%c-string: expressions nested too deeply", TOK_GET_STRING_PREFIX(tok))); - } - TOK_GET_MODE(tok)->kind = TOK_REGULAR_MODE; - return tok_get_normal_mode(tok, current_tok, token); - } - } - else { - tok_backup(tok, start_char); - } - - // Check if we are at the end of the string - for (int i = 0; i < current_tok->quote_size; i++) { - int quote = tok_nextc(tok); - if (quote != current_tok->quote) { - tok_backup(tok, quote); - goto f_string_middle; - } - } - - if (current_tok->last_expr_buffer != NULL) { - PyMem_Free(current_tok->last_expr_buffer); - current_tok->last_expr_buffer = NULL; - current_tok->last_expr_size = 0; - current_tok->last_expr_end = -1; - } - - p_start = tok->start; - p_end = tok->cur; - tok->tok_mode_stack_index--; - return MAKE_TOKEN(FTSTRING_END(current_tok)); - -f_string_middle: - - // TODO: This is a bit of a hack, but it works for now. We need to find a better way to handle - // this. - tok->multi_line_start = tok->line_start; - while (end_quote_size != current_tok->quote_size) { - int c = tok_nextc(tok); - if (tok->done == E_ERROR || tok->done == E_DECODE) { - return MAKE_TOKEN(ERRORTOKEN); - } - int in_format_spec = ( - current_tok->in_format_spec - && - INSIDE_FSTRING_EXPR(current_tok) - ); - - if (c == EOF || (current_tok->quote_size == 1 && c == '\n')) { - if (tok->decoding_erred) { - return MAKE_TOKEN(ERRORTOKEN); - } - - // If we are in a format spec and we found a newline, - // it means that the format spec ends here and we should - // return to the regular mode. - if (in_format_spec && c == '\n') { - if (current_tok->quote_size == 1) { - return MAKE_TOKEN( - _PyTokenizer_syntaxerror( - tok, - "%c-string: newlines are not allowed in format specifiers for single quoted %c-strings", - TOK_GET_STRING_PREFIX(tok), TOK_GET_STRING_PREFIX(tok) - ) - ); - } - tok_backup(tok, c); - TOK_GET_MODE(tok)->kind = TOK_REGULAR_MODE; - current_tok->in_format_spec = 0; - p_start = tok->start; - p_end = tok->cur; - return MAKE_TOKEN(FTSTRING_MIDDLE(current_tok)); - } - - assert(tok->multi_line_start != NULL); - // shift the tok_state's location into - // the start of string, and report the error - // from the initial quote character - tok->cur = (char *)current_tok->start; - tok->cur++; - tok->line_start = current_tok->multi_line_start; - int start = tok->lineno; - - tokenizer_mode *the_current_tok = TOK_GET_MODE(tok); - tok->lineno = the_current_tok->first_line; - - if (current_tok->quote_size == 3) { - _PyTokenizer_syntaxerror(tok, - "unterminated triple-quoted %c-string literal" - " (detected at line %d)", - TOK_GET_STRING_PREFIX(tok), start); - if (c != '\n') { - tok->done = E_EOFS; - } - return MAKE_TOKEN(ERRORTOKEN); - } - else { - return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, - "unterminated %c-string literal (detected at" - " line %d)", TOK_GET_STRING_PREFIX(tok), start)); - } - } - - if (c == current_tok->quote) { - end_quote_size += 1; - continue; - } else { - end_quote_size = 0; - } - - if (c == '{') { - if (!_PyLexer_update_ftstring_expr(tok, c)) { - return MAKE_TOKEN(ENDMARKER); - } - int peek = tok_nextc(tok); - if (peek != '{' || in_format_spec) { - tok_backup(tok, peek); - tok_backup(tok, c); - current_tok->curly_bracket_expr_start_depth++; - if (current_tok->curly_bracket_expr_start_depth >= MAX_EXPR_NESTING) { - return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, - "%c-string: expressions nested too deeply", TOK_GET_STRING_PREFIX(tok))); - } - TOK_GET_MODE(tok)->kind = TOK_REGULAR_MODE; - current_tok->in_format_spec = 0; - p_start = tok->start; - p_end = tok->cur; - } else { - p_start = tok->start; - p_end = tok->cur - 1; - } - return MAKE_TOKEN(FTSTRING_MIDDLE(current_tok)); - } else if (c == '}') { - if (unicode_escape) { - p_start = tok->start; - p_end = tok->cur; - return MAKE_TOKEN(FTSTRING_MIDDLE(current_tok)); - } - int peek = tok_nextc(tok); - - // The tokenizer can only be in the format spec if we have already completed the expression - // scanning (indicated by the end of the expression being set) and we are not at the top level - // of the bracket stack (-1 is the top level). Since format specifiers can't legally use double - // brackets, we can bypass it here. - int cursor = current_tok->curly_bracket_depth; - if (peek == '}' && !in_format_spec && cursor == 0) { - p_start = tok->start; - p_end = tok->cur - 1; - } else { - tok_backup(tok, peek); - tok_backup(tok, c); - TOK_GET_MODE(tok)->kind = TOK_REGULAR_MODE; - current_tok->in_format_spec = 0; - p_start = tok->start; - p_end = tok->cur; - } - return MAKE_TOKEN(FTSTRING_MIDDLE(current_tok)); - } else if (c == '\\') { - int peek = tok_nextc(tok); - if (peek == '\r') { - peek = tok_nextc(tok); - } - // Special case when the backslash is right before a curly - // brace. We have to restore and return the control back - // to the loop for the next iteration. - if (peek == '{' || peek == '}') { - if (!current_tok->raw) { - if (_PyTokenizer_warn_invalid_escape_sequence(tok, peek)) { - return MAKE_TOKEN(ERRORTOKEN); - } - } - tok_backup(tok, peek); - continue; - } - - if (!current_tok->raw) { - if (peek == 'N') { - /* Handle named unicode escapes (\N{BULLET}) */ - peek = tok_nextc(tok); - if (peek == '{') { - unicode_escape = 1; - } else { - tok_backup(tok, peek); - } - } - } /* else { - skip the escaped character - }*/ - } - } - - // Backup the f-string quotes to emit a final FSTRING_MIDDLE and - // add the quotes to the FSTRING_END in the next tokenizer iteration. - for (int i = 0; i < current_tok->quote_size; i++) { - tok_backup(tok, current_tok->quote); - } - p_start = tok->start; - p_end = tok->cur; - return MAKE_TOKEN(FTSTRING_MIDDLE(current_tok)); -} static int tok_get(struct tok_state *tok, struct token *token) { tokenizer_mode *current_tok = TOK_GET_MODE(tok); if (current_tok->kind == TOK_REGULAR_MODE) { - return tok_get_normal_mode(tok, current_tok, token); + return _PyLexer_get_normal_mode(tok, current_tok, token); } else { - return tok_get_fstring_mode(tok, current_tok, token); + return _PyLexer_get_fstring_mode(tok, current_tok, token); } } diff --git a/Parser/lexer/lexer_internal.h b/Parser/lexer/lexer_internal.h new file mode 100644 index 00000000000000..c6d3b9045c7292 --- /dev/null +++ b/Parser/lexer/lexer_internal.h @@ -0,0 +1,57 @@ +#ifndef _PY_LEXER_INTERNAL_H_ +#define _PY_LEXER_INTERNAL_H_ + +#include "lexer.h" + +#define is_potential_identifier_start(c) (\ + (c >= 'a' && c <= 'z')\ + || (c >= 'A' && c <= 'Z')\ + || c == '_'\ + || (c >= 128)) + +#define is_potential_identifier_char(c) (\ + (c >= 'a' && c <= 'z')\ + || (c >= 'A' && c <= 'Z')\ + || (c >= '0' && c <= '9')\ + || c == '_'\ + || (c >= 128)) + +#ifdef Py_DEBUG +static inline tokenizer_mode * +TOK_GET_MODE(struct tok_state *tok) +{ + assert(tok->tok_mode_stack_index >= 0); + assert(tok->tok_mode_stack_index < MAXFSTRINGLEVEL); + return &tok->tok_mode_stack[tok->tok_mode_stack_index]; +} + +static inline tokenizer_mode * +TOK_NEXT_MODE(struct tok_state *tok) +{ + assert(tok->tok_mode_stack_index >= 0); + assert(tok->tok_mode_stack_index + 1 < MAXFSTRINGLEVEL); + return &tok->tok_mode_stack[++tok->tok_mode_stack_index]; +} +#else +#define TOK_GET_MODE(tok) (&(tok)->tok_mode_stack[(tok)->tok_mode_stack_index]) +#define TOK_NEXT_MODE(tok) (&(tok)->tok_mode_stack[++(tok)->tok_mode_stack_index]) +#endif + +#define FTSTRING_MIDDLE(tok_mode) ((tok_mode)->string_kind == TSTRING ? TSTRING_MIDDLE : FSTRING_MIDDLE) +#define FTSTRING_END(tok_mode) ((tok_mode)->string_kind == TSTRING ? TSTRING_END : FSTRING_END) +#define TOK_GET_STRING_PREFIX(tok) (TOK_GET_MODE(tok)->string_kind == TSTRING ? 't' : 'f') + +#define tok_nextc _PyLexer_nextc +#define tok_backup _PyLexer_backup + +int _PyLexer_nextc(struct tok_state *); +void _PyLexer_backup(struct tok_state *, int); +int _PyLexer_set_ftstring_expr(struct tok_state *, struct token *, char); +int _PyLexer_check_string_prefixes(struct tok_state *, int, int, int, int, int); +int _PyLexer_scan_number(struct tok_state *, struct token *, int, int); +int _PyLexer_scan_fstring_start(struct tok_state *, struct token *, int); +int _PyLexer_scan_string(struct tok_state *, struct token *, int); +int _PyLexer_get_normal_mode(struct tok_state *, tokenizer_mode *, struct token *); +int _PyLexer_get_fstring_mode(struct tok_state *, tokenizer_mode *, struct token *); + +#endif diff --git a/Parser/lexer/number.c b/Parser/lexer/number.c new file mode 100644 index 00000000000000..8bca8cbb9adfe5 --- /dev/null +++ b/Parser/lexer/number.c @@ -0,0 +1,313 @@ +#include "Python.h" +#include "pycore_token.h" + +#include "lexer_internal.h" +#include "../tokenizer/helpers.h" + +#define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end) + +static int +lookahead(struct tok_state *tok, const char *test) +{ + const char *s = test; + int res = 0; + while (1) { + int c = tok_nextc(tok); + if (*s == 0) { + res = !is_potential_identifier_char(c); + } + else if (c == *s) { + s++; + continue; + } + + tok_backup(tok, c); + while (s != test) { + tok_backup(tok, *--s); + } + return res; + } +} + +static int +verify_end_of_number(struct tok_state *tok, int c, const char *kind) { + if (tok->tok_extra_tokens) { + // When we are parsing extra tokens, we don't want to emit warnings + // about invalid literals, because we want to be a bit more liberal. + return 1; + } + /* Emit a deprecation warning only if the numeric literal is immediately + * followed by one of keywords which can occur after a numeric literal + * in valid code: "and", "else", "for", "if", "in", "is" and "or". + * It allows to gradually deprecate existing valid code without adding + * warning before error in most cases of invalid numeric literal (which + * would be confusing and break existing tests). + * Raise a syntax error with slightly better message than plain + * "invalid syntax" if the numeric literal is immediately followed by + * other keyword or identifier. + */ + int r = 0; + if (c == 'a') { + r = lookahead(tok, "nd"); + } + else if (c == 'e') { + r = lookahead(tok, "lse"); + } + else if (c == 'f') { + r = lookahead(tok, "or"); + } + else if (c == 'i') { + int c2 = tok_nextc(tok); + if (c2 == 'f' || c2 == 'n' || c2 == 's') { + r = 1; + } + tok_backup(tok, c2); + } + else if (c == 'o') { + r = lookahead(tok, "r"); + } + else if (c == 'n') { + r = lookahead(tok, "ot"); + } + if (r) { + tok_backup(tok, c); + if (_PyTokenizer_parser_warn(tok, PyExc_SyntaxWarning, + "invalid %s literal", kind)) + { + return 0; + } + tok_nextc(tok); + } + else /* In future releases, only error will remain. */ + if (c < 128 && is_potential_identifier_char(c)) { + tok_backup(tok, c); + _PyTokenizer_syntaxerror(tok, "invalid %s literal", kind); + return 0; + } + return 1; +} + +static int +tok_decimal_tail(struct tok_state *tok) +{ + int c; + + while (1) { + do { + c = tok_nextc(tok); + } while (Py_ISDIGIT(c)); + if (c != '_') { + break; + } + c = tok_nextc(tok); + if (!Py_ISDIGIT(c)) { + tok_backup(tok, c); + _PyTokenizer_syntaxerror(tok, "invalid decimal literal"); + return 0; + } + } + return c; +} + +int +_PyLexer_scan_number(struct tok_state *tok, struct token *token, int c, + int leading_dot) +{ + const char *p_start = NULL; + const char *p_end = NULL; + + if (leading_dot) { + goto fraction; + } + if (c == '0') { + /* Hex, octal or binary -- maybe. */ + c = tok_nextc(tok); + if (c == 'x' || c == 'X') { + /* Hex */ + c = tok_nextc(tok); + do { + if (c == '_') { + c = tok_nextc(tok); + } + if (!Py_ISXDIGIT(c)) { + tok_backup(tok, c); + return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid hexadecimal literal")); + } + do { + c = tok_nextc(tok); + } while (Py_ISXDIGIT(c)); + } while (c == '_'); + if (!verify_end_of_number(tok, c, "hexadecimal")) { + return MAKE_TOKEN(ERRORTOKEN); + } + } + else if (c == 'o' || c == 'O') { + /* Octal */ + c = tok_nextc(tok); + do { + if (c == '_') { + c = tok_nextc(tok); + } + if (c < '0' || c >= '8') { + if (Py_ISDIGIT(c)) { + return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, + "invalid digit '%c' in octal literal", c)); + } + else { + tok_backup(tok, c); + return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid octal literal")); + } + } + do { + c = tok_nextc(tok); + } while ('0' <= c && c < '8'); + } while (c == '_'); + if (Py_ISDIGIT(c)) { + return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, + "invalid digit '%c' in octal literal", c)); + } + if (!verify_end_of_number(tok, c, "octal")) { + return MAKE_TOKEN(ERRORTOKEN); + } + } + else if (c == 'b' || c == 'B') { + /* Binary */ + c = tok_nextc(tok); + do { + if (c == '_') { + c = tok_nextc(tok); + } + if (c != '0' && c != '1') { + if (Py_ISDIGIT(c)) { + return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid digit '%c' in binary literal", c)); + } + else { + tok_backup(tok, c); + return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid binary literal")); + } + } + do { + c = tok_nextc(tok); + } while (c == '0' || c == '1'); + } while (c == '_'); + if (Py_ISDIGIT(c)) { + return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid digit '%c' in binary literal", c)); + } + if (!verify_end_of_number(tok, c, "binary")) { + return MAKE_TOKEN(ERRORTOKEN); + } + } + else { + int nonzero = 0; + /* maybe old-style octal; c is first char of it */ + /* in any case, allow '0' as a literal */ + while (1) { + if (c == '_') { + c = tok_nextc(tok); + if (!Py_ISDIGIT(c)) { + tok_backup(tok, c); + return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid decimal literal")); + } + } + if (c != '0') { + break; + } + c = tok_nextc(tok); + } + char* zeros_end = tok->cur; + if (Py_ISDIGIT(c)) { + nonzero = 1; + c = tok_decimal_tail(tok); + if (c == 0) { + return MAKE_TOKEN(ERRORTOKEN); + } + } + if (c == '.') { + c = tok_nextc(tok); + goto fraction; + } + else if (c == 'e' || c == 'E') { + goto exponent; + } + else if (c == 'j' || c == 'J') { + goto imaginary; + } + else if (nonzero && !tok->tok_extra_tokens) { + /* Old-style octal: now disallowed. */ + tok_backup(tok, c); + return MAKE_TOKEN(_PyTokenizer_syntaxerror_known_range( + tok, (int)(tok->start + 1 - tok->line_start), + (int)(zeros_end - tok->line_start), + "leading zeros in decimal integer " + "literals are not permitted; " + "use an 0o prefix for octal integers")); + } + if (!verify_end_of_number(tok, c, "decimal")) { + return MAKE_TOKEN(ERRORTOKEN); + } + } + } + else { + /* Decimal */ + c = tok_decimal_tail(tok); + if (c == 0) { + return MAKE_TOKEN(ERRORTOKEN); + } + { + /* Accept floating-point numbers. */ + if (c == '.') { + c = tok_nextc(tok); + fraction: + /* Fraction */ + if (Py_ISDIGIT(c)) { + c = tok_decimal_tail(tok); + if (c == 0) { + return MAKE_TOKEN(ERRORTOKEN); + } + } + } + if (c == 'e' || c == 'E') { + int e; + exponent: + e = c; + /* Exponent part */ + c = tok_nextc(tok); + if (c == '+' || c == '-') { + c = tok_nextc(tok); + if (!Py_ISDIGIT(c)) { + tok_backup(tok, c); + return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid decimal literal")); + } + } else if (!Py_ISDIGIT(c)) { + tok_backup(tok, c); + if (!verify_end_of_number(tok, e, "decimal")) { + return MAKE_TOKEN(ERRORTOKEN); + } + tok_backup(tok, e); + p_start = tok->start; + p_end = tok->cur; + return MAKE_TOKEN(NUMBER); + } + c = tok_decimal_tail(tok); + if (c == 0) { + return MAKE_TOKEN(ERRORTOKEN); + } + } + if (c == 'j' || c == 'J') { + /* Imaginary part */ + imaginary: + c = tok_nextc(tok); + if (!verify_end_of_number(tok, c, "imaginary")) { + return MAKE_TOKEN(ERRORTOKEN); + } + } + else if (!verify_end_of_number(tok, c, "decimal")) { + return MAKE_TOKEN(ERRORTOKEN); + } + } + } + tok_backup(tok, c); + p_start = tok->start; + p_end = tok->cur; + return MAKE_TOKEN(NUMBER); +} diff --git a/Parser/lexer/string.c b/Parser/lexer/string.c new file mode 100644 index 00000000000000..e546954ba5c0d8 --- /dev/null +++ b/Parser/lexer/string.c @@ -0,0 +1,642 @@ +#include "Python.h" +#include "pycore_token.h" +#include "errcode.h" + +#include "lexer_internal.h" +#include "../tokenizer/helpers.h" + +#define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end) + +int +_PyLexer_set_ftstring_expr(struct tok_state* tok, struct token *token, char c) { + assert(token != NULL); + assert(c == '}' || c == ':' || c == '!'); + tokenizer_mode *tok_mode = TOK_GET_MODE(tok); + + if (!(tok_mode->in_debug || tok_mode->string_kind == TSTRING) || token->metadata) { + return 0; + } + PyObject *res = NULL; + + // Look for a # character outside of string literals + int hash_detected = 0; + int in_string = 0; + char quote_char = 0; + + for (Py_ssize_t i = 0; i < tok_mode->last_expr_size - tok_mode->last_expr_end; i++) { + char ch = tok_mode->last_expr_buffer[i]; + + // Skip escaped characters + if (ch == '\\') { + i++; + continue; + } + + // Handle quotes + if (ch == '"' || ch == '\'') { + // The following if/else block works becase there is an off number + // of quotes in STRING tokens and the lexer only ever reaches this + // function with valid STRING tokens. + // For example: """hello""" + // First quote: in_string = 1 + // Second quote: in_string = 0 + // Third quote: in_string = 1 + if (!in_string) { + in_string = 1; + quote_char = ch; + } + else if (ch == quote_char) { + in_string = 0; + } + continue; + } + + // Check for # outside strings + if (ch == '#' && !in_string) { + hash_detected = 1; + break; + } + } + // If we found a # character in the expression, we need to handle comments + if (hash_detected) { + // Allocate buffer for processed result + char *result = (char *)PyMem_Malloc((tok_mode->last_expr_size - tok_mode->last_expr_end + 1) * sizeof(char)); + if (!result) { + return -1; + } + + Py_ssize_t i = 0; // Input position + Py_ssize_t j = 0; // Output position + in_string = 0; // Whether we're in a string + quote_char = 0; // Current string quote char + + // Process each character + while (i < tok_mode->last_expr_size - tok_mode->last_expr_end) { + char ch = tok_mode->last_expr_buffer[i]; + + // Handle string quotes + if (ch == '"' || ch == '\'') { + // See comment above to understand this part + if (!in_string) { + in_string = 1; + quote_char = ch; + } else if (ch == quote_char) { + in_string = 0; + } + result[j++] = ch; + } + // Skip comments + else if (ch == '#' && !in_string) { + while (i < tok_mode->last_expr_size - tok_mode->last_expr_end && + tok_mode->last_expr_buffer[i] != '\n') { + i++; + } + if (i < tok_mode->last_expr_size - tok_mode->last_expr_end) { + result[j++] = '\n'; + } + } + // Copy other chars + else { + result[j++] = ch; + } + i++; + } + + result[j] = '\0'; // Null-terminate the result string + res = PyUnicode_DecodeUTF8(result, j, NULL); + PyMem_Free(result); + } else { + res = PyUnicode_DecodeUTF8( + tok_mode->last_expr_buffer, + tok_mode->last_expr_size - tok_mode->last_expr_end, + NULL + ); + } + + if (!res) { + return -1; + } + token->metadata = res; + return 0; +} + +int +_PyLexer_update_ftstring_expr(struct tok_state *tok, char cur) +{ + assert(tok->cur != NULL); + + Py_ssize_t size = strlen(tok->cur); + tokenizer_mode *tok_mode = TOK_GET_MODE(tok); + + switch (cur) { + case 0: + if (!tok_mode->last_expr_buffer || tok_mode->last_expr_end >= 0) { + return 1; + } + char *new_buffer = PyMem_Realloc( + tok_mode->last_expr_buffer, + tok_mode->last_expr_size + size + ); + if (new_buffer == NULL) { + PyMem_Free(tok_mode->last_expr_buffer); + goto error; + } + tok_mode->last_expr_buffer = new_buffer; + strncpy(tok_mode->last_expr_buffer + tok_mode->last_expr_size, tok->cur, size); + tok_mode->last_expr_size += size; + break; + case '{': + if (tok_mode->last_expr_buffer != NULL) { + PyMem_Free(tok_mode->last_expr_buffer); + } + tok_mode->last_expr_buffer = PyMem_Malloc(size); + if (tok_mode->last_expr_buffer == NULL) { + goto error; + } + tok_mode->last_expr_size = size; + tok_mode->last_expr_end = -1; + strncpy(tok_mode->last_expr_buffer, tok->cur, size); + break; + case '}': + case '!': + tok_mode->last_expr_end = strlen(tok->start); + break; + case ':': + if (tok_mode->last_expr_end == -1) { + tok_mode->last_expr_end = strlen(tok->start); + } + break; + default: + Py_UNREACHABLE(); + } + return 1; +error: + tok->done = E_NOMEM; + return 0; +} + +int +_PyLexer_check_string_prefixes(struct tok_state *tok, + int saw_b, int saw_r, int saw_u, + int saw_f, int saw_t) { + // Supported: rb, rf, rt (in any order) + // Unsupported: ub, ur, uf, ut, bf, bt, ft (in any order) + +#define RETURN_SYNTAX_ERROR(PREFIX1, PREFIX2) \ + do { \ + (void)_PyTokenizer_syntaxerror_known_range( \ + tok, (int)(tok->start + 1 - tok->line_start), \ + (int)(tok->cur - tok->line_start), \ + "'" PREFIX1 "' and '" PREFIX2 "' prefixes are incompatible"); \ + return -1; \ + } while (0) + + if (saw_u && saw_b) { + RETURN_SYNTAX_ERROR("u", "b"); + } + if (saw_u && saw_r) { + RETURN_SYNTAX_ERROR("u", "r"); + } + if (saw_u && saw_f) { + RETURN_SYNTAX_ERROR("u", "f"); + } + if (saw_u && saw_t) { + RETURN_SYNTAX_ERROR("u", "t"); + } + + if (saw_b && saw_f) { + RETURN_SYNTAX_ERROR("b", "f"); + } + if (saw_b && saw_t) { + RETURN_SYNTAX_ERROR("b", "t"); + } + + if (saw_f && saw_t) { + RETURN_SYNTAX_ERROR("f", "t"); + } + +#undef RETURN_SYNTAX_ERROR + + return 0; +} + +int +_PyLexer_scan_fstring_start(struct tok_state *tok, struct token *token, int c) +{ + const char *p_start = NULL; + const char *p_end = NULL; + + int quote = c; + int quote_size = 1; /* 1 or 3 */ + + /* Nodes of type STRING, especially multi line strings + must be handled differently in order to get both + the starting line number and the column offset right. + (cf. issue 16806) */ + tok->first_lineno = tok->lineno; + tok->multi_line_start = tok->line_start; + + /* Find the quote size and start of string */ + int after_quote = tok_nextc(tok); + if (after_quote == quote) { + int after_after_quote = tok_nextc(tok); + if (after_after_quote == quote) { + quote_size = 3; + } + else { + // TODO: Check this + tok_backup(tok, after_after_quote); + tok_backup(tok, after_quote); + } + } + if (after_quote != quote) { + tok_backup(tok, after_quote); + } + + + p_start = tok->start; + p_end = tok->cur; + if (tok->tok_mode_stack_index + 1 >= MAXFSTRINGLEVEL) { + return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "too many nested f-strings or t-strings")); + } + tokenizer_mode *the_current_tok = TOK_NEXT_MODE(tok); + the_current_tok->kind = TOK_FSTRING_MODE; + the_current_tok->quote = quote; + the_current_tok->quote_size = quote_size; + the_current_tok->start = tok->start; + the_current_tok->multi_line_start = tok->line_start; + the_current_tok->first_line = tok->lineno; + the_current_tok->start_offset = -1; + the_current_tok->multi_line_start_offset = -1; + the_current_tok->last_expr_buffer = NULL; + the_current_tok->last_expr_size = 0; + the_current_tok->last_expr_end = -1; + the_current_tok->in_format_spec = 0; + the_current_tok->in_debug = 0; + + enum string_kind_t string_kind = FSTRING; + switch (*tok->start) { + case 'T': + case 't': + the_current_tok->raw = Py_TOLOWER(*(tok->start + 1)) == 'r'; + string_kind = TSTRING; + break; + case 'F': + case 'f': + the_current_tok->raw = Py_TOLOWER(*(tok->start + 1)) == 'r'; + break; + case 'R': + case 'r': + the_current_tok->raw = 1; + if (Py_TOLOWER(*(tok->start + 1)) == 't') { + string_kind = TSTRING; + } + break; + default: + Py_UNREACHABLE(); + } + + the_current_tok->string_kind = string_kind; + the_current_tok->curly_bracket_depth = 0; + the_current_tok->curly_bracket_expr_start_depth = -1; + return string_kind == TSTRING ? MAKE_TOKEN(TSTRING_START) : MAKE_TOKEN(FSTRING_START); +} + +int +_PyLexer_scan_string(struct tok_state *tok, struct token *token, int c) +{ + const char *p_start = NULL; + const char *p_end = NULL; + + int quote = c; + int quote_size = 1; /* 1 or 3 */ + int end_quote_size = 0; + int has_escaped_quote = 0; + + /* Nodes of type STRING, especially multi line strings + must be handled differently in order to get both + the starting line number and the column offset right. + (cf. issue 16806) */ + tok->first_lineno = tok->lineno; + tok->multi_line_start = tok->line_start; + + /* Find the quote size and start of string */ + c = tok_nextc(tok); + if (c == quote) { + c = tok_nextc(tok); + if (c == quote) { + quote_size = 3; + } + else { + end_quote_size = 1; /* empty string found */ + } + } + if (c != quote) { + tok_backup(tok, c); + } + + /* Get rest of string */ + while (end_quote_size != quote_size) { + c = tok_nextc(tok); + if (tok->done == E_ERROR) { + return MAKE_TOKEN(ERRORTOKEN); + } + if (tok->done == E_DECODE) { + break; + } + if (c == EOF || (quote_size == 1 && c == '\n')) { + assert(tok->multi_line_start != NULL); + // shift the tok_state's location into + // the start of string, and report the error + // from the initial quote character + tok->cur = (char *)tok->start; + tok->cur++; + tok->line_start = tok->multi_line_start; + int start = tok->lineno; + tok->lineno = tok->first_lineno; + + if (INSIDE_FSTRING(tok)) { + /* When we are in an f-string, before raising the + * unterminated string literal error, check whether + * does the initial quote matches with f-strings quotes + * and if it is, then this must be a missing '}' token + * so raise the proper error */ + tokenizer_mode *the_current_tok = TOK_GET_MODE(tok); + if (the_current_tok->quote == quote && + the_current_tok->quote_size == quote_size) { + return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, + "%c-string: expecting '}'", TOK_GET_STRING_PREFIX(tok))); + } + } + + if (quote_size == 3) { + _PyTokenizer_syntaxerror(tok, "unterminated triple-quoted string literal" + " (detected at line %d)", start); + if (c != '\n') { + tok->done = E_EOFS; + } + return MAKE_TOKEN(ERRORTOKEN); + } + else { + if (has_escaped_quote) { + _PyTokenizer_syntaxerror( + tok, + "unterminated string literal (detected at line %d); " + "perhaps you escaped the end quote?", + start + ); + } else { + _PyTokenizer_syntaxerror( + tok, "unterminated string literal (detected at line %d)", start + ); + } + if (c != '\n') { + tok->done = E_EOLS; + } + return MAKE_TOKEN(ERRORTOKEN); + } + } + if (c == quote) { + end_quote_size += 1; + } + else { + end_quote_size = 0; + if (c == '\\') { + c = tok_nextc(tok); /* skip escaped char */ + if (c == quote) { /* but record whether the escaped char was a quote */ + has_escaped_quote = 1; + } + if (c == '\r') { + c = tok_nextc(tok); + } + } + } + } + + p_start = tok->start; + p_end = tok->cur; + return MAKE_TOKEN(STRING); +} + +int +_PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct token *token) +{ + const char *p_start = NULL; + const char *p_end = NULL; + int end_quote_size = 0; + int unicode_escape = 0; + + tok->start = tok->cur; + tok->first_lineno = tok->lineno; + tok->starting_col_offset = tok->col_offset; + + // If we start with a bracket, we defer to the normal mode as there is nothing for us to tokenize + // before it. + int start_char = tok_nextc(tok); + if (start_char == '{') { + int peek1 = tok_nextc(tok); + tok_backup(tok, peek1); + tok_backup(tok, start_char); + if (peek1 != '{') { + current_tok->curly_bracket_expr_start_depth++; + if (current_tok->curly_bracket_expr_start_depth >= MAX_EXPR_NESTING) { + return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, + "%c-string: expressions nested too deeply", TOK_GET_STRING_PREFIX(tok))); + } + TOK_GET_MODE(tok)->kind = TOK_REGULAR_MODE; + return _PyLexer_get_normal_mode(tok, current_tok, token); + } + } + else { + tok_backup(tok, start_char); + } + + // Check if we are at the end of the string + for (int i = 0; i < current_tok->quote_size; i++) { + int quote = tok_nextc(tok); + if (quote != current_tok->quote) { + tok_backup(tok, quote); + goto f_string_middle; + } + } + + if (current_tok->last_expr_buffer != NULL) { + PyMem_Free(current_tok->last_expr_buffer); + current_tok->last_expr_buffer = NULL; + current_tok->last_expr_size = 0; + current_tok->last_expr_end = -1; + } + + p_start = tok->start; + p_end = tok->cur; + tok->tok_mode_stack_index--; + return MAKE_TOKEN(FTSTRING_END(current_tok)); + +f_string_middle: + + // TODO: This is a bit of a hack, but it works for now. We need to find a better way to handle + // this. + tok->multi_line_start = tok->line_start; + while (end_quote_size != current_tok->quote_size) { + int c = tok_nextc(tok); + if (tok->done == E_ERROR || tok->done == E_DECODE) { + return MAKE_TOKEN(ERRORTOKEN); + } + int in_format_spec = ( + current_tok->in_format_spec + && + INSIDE_FSTRING_EXPR(current_tok) + ); + + if (c == EOF || (current_tok->quote_size == 1 && c == '\n')) { + if (tok->decoding_erred) { + return MAKE_TOKEN(ERRORTOKEN); + } + + // If we are in a format spec and we found a newline, + // it means that the format spec ends here and we should + // return to the regular mode. + if (in_format_spec && c == '\n') { + if (current_tok->quote_size == 1) { + return MAKE_TOKEN( + _PyTokenizer_syntaxerror( + tok, + "%c-string: newlines are not allowed in format specifiers for single quoted %c-strings", + TOK_GET_STRING_PREFIX(tok), TOK_GET_STRING_PREFIX(tok) + ) + ); + } + tok_backup(tok, c); + TOK_GET_MODE(tok)->kind = TOK_REGULAR_MODE; + current_tok->in_format_spec = 0; + p_start = tok->start; + p_end = tok->cur; + return MAKE_TOKEN(FTSTRING_MIDDLE(current_tok)); + } + + assert(tok->multi_line_start != NULL); + // shift the tok_state's location into + // the start of string, and report the error + // from the initial quote character + tok->cur = (char *)current_tok->start; + tok->cur++; + tok->line_start = current_tok->multi_line_start; + int start = tok->lineno; + + tokenizer_mode *the_current_tok = TOK_GET_MODE(tok); + tok->lineno = the_current_tok->first_line; + + if (current_tok->quote_size == 3) { + _PyTokenizer_syntaxerror(tok, + "unterminated triple-quoted %c-string literal" + " (detected at line %d)", + TOK_GET_STRING_PREFIX(tok), start); + if (c != '\n') { + tok->done = E_EOFS; + } + return MAKE_TOKEN(ERRORTOKEN); + } + else { + return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, + "unterminated %c-string literal (detected at" + " line %d)", TOK_GET_STRING_PREFIX(tok), start)); + } + } + + if (c == current_tok->quote) { + end_quote_size += 1; + continue; + } else { + end_quote_size = 0; + } + + if (c == '{') { + if (!_PyLexer_update_ftstring_expr(tok, c)) { + return MAKE_TOKEN(ENDMARKER); + } + int peek = tok_nextc(tok); + if (peek != '{' || in_format_spec) { + tok_backup(tok, peek); + tok_backup(tok, c); + current_tok->curly_bracket_expr_start_depth++; + if (current_tok->curly_bracket_expr_start_depth >= MAX_EXPR_NESTING) { + return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, + "%c-string: expressions nested too deeply", TOK_GET_STRING_PREFIX(tok))); + } + TOK_GET_MODE(tok)->kind = TOK_REGULAR_MODE; + current_tok->in_format_spec = 0; + p_start = tok->start; + p_end = tok->cur; + } else { + p_start = tok->start; + p_end = tok->cur - 1; + } + return MAKE_TOKEN(FTSTRING_MIDDLE(current_tok)); + } else if (c == '}') { + if (unicode_escape) { + p_start = tok->start; + p_end = tok->cur; + return MAKE_TOKEN(FTSTRING_MIDDLE(current_tok)); + } + int peek = tok_nextc(tok); + + // The tokenizer can only be in the format spec if we have already completed the expression + // scanning (indicated by the end of the expression being set) and we are not at the top level + // of the bracket stack (-1 is the top level). Since format specifiers can't legally use double + // brackets, we can bypass it here. + int cursor = current_tok->curly_bracket_depth; + if (peek == '}' && !in_format_spec && cursor == 0) { + p_start = tok->start; + p_end = tok->cur - 1; + } else { + tok_backup(tok, peek); + tok_backup(tok, c); + TOK_GET_MODE(tok)->kind = TOK_REGULAR_MODE; + current_tok->in_format_spec = 0; + p_start = tok->start; + p_end = tok->cur; + } + return MAKE_TOKEN(FTSTRING_MIDDLE(current_tok)); + } else if (c == '\\') { + int peek = tok_nextc(tok); + if (peek == '\r') { + peek = tok_nextc(tok); + } + // Special case when the backslash is right before a curly + // brace. We have to restore and return the control back + // to the loop for the next iteration. + if (peek == '{' || peek == '}') { + if (!current_tok->raw) { + if (_PyTokenizer_warn_invalid_escape_sequence(tok, peek)) { + return MAKE_TOKEN(ERRORTOKEN); + } + } + tok_backup(tok, peek); + continue; + } + + if (!current_tok->raw) { + if (peek == 'N') { + /* Handle named unicode escapes (\N{BULLET}) */ + peek = tok_nextc(tok); + if (peek == '{') { + unicode_escape = 1; + } else { + tok_backup(tok, peek); + } + } + } /* else { + skip the escaped character + }*/ + } + } + + // Backup the f-string quotes to emit a final FSTRING_MIDDLE and + // add the quotes to the FSTRING_END in the next tokenizer iteration. + for (int i = 0; i < current_tok->quote_size; i++) { + tok_backup(tok, current_tok->quote); + } + p_start = tok->start; + p_end = tok->cur; + return MAKE_TOKEN(FTSTRING_MIDDLE(current_tok)); +} diff --git a/Tools/peg_generator/pegen/build.py b/Tools/peg_generator/pegen/build.py index bc3657067a8da2..37883afeffcb47 100644 --- a/Tools/peg_generator/pegen/build.py +++ b/Tools/peg_generator/pegen/build.py @@ -125,7 +125,9 @@ def compile_c_extension( str(MOD_DIR.parent.parent.parent / "Python" / "Python-ast.c"), str(MOD_DIR.parent.parent.parent / "Python" / "asdl.c"), str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "lexer.c"), + str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "number.c"), str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "state.c"), + str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "string.c"), str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "buffer.c"), str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" / "string_tokenizer.c"), str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" / "file_tokenizer.c"),