gh-153569: keep active formatted-string frames

Author: pablogsal

Lib/test/test_fstring.py

@@ -1657,6 +1657,7 @@ def __repr__(self):
         self.assertEqual(f'{C()=:x}', 'C()=FORMAT-x')
         self.assertEqual(f'{C()=!r:*^20}', 'C()=********REPR********')
         self.assertEqual(f"{C():{20=}}", 'FORMAT-20=20')
+        self.assertEqual(f"{C():{C():{4=}}}", 'FORMAT-FORMAT-4=4')
 
         self.assertRaises(SyntaxError, eval, "f'{C=]'")
 

Lib/test/test_tokenize.py

@@ -2592,6 +2592,21 @@ def test_incomplete_formatted_string_comment_after_carriage_return(self):
                         ("unexpected EOF in multi-line statement", (1, 7)),
                     )
 
+    def test_formatted_string_nesting_limit(self):
+        def nested_string(depth, prefix):
+            source = "'x'"
+            for _ in range(depth):
+                source = f'{prefix}"{{{source}}}"'
+            return source
+
+        for prefix in ("f", "t"):
+            with self.subTest(prefix=prefix):
+                self._get_tokens(nested_string(149, prefix))
+                with self.assertRaisesRegex(
+                        tokenize.TokenError,
+                        "too many nested f-strings or t-strings"):
+                    self._get_tokens(nested_string(150, prefix))
+
     def test_escaped_fstring_brace_has_a_position_gap(self):
         tokens = self._get_tokens('f"a{{"', extra_tokens=True)
         self.assertEqual(

Lib/test/test_tstring.py

@@ -140,6 +140,15 @@ def test_debug_specifier(self):
         )
         self.assertEqual(fstring(t), "Value: value = 42")
 
+        class C:
+            def __format__(self, spec):
+                return f"FORMAT-{spec}"
+
+        x = y = C()
+        t = t"{x:{y:{value=}}}"
+        self.assertEqual(t.interpolations[0].format_spec,
+                         "FORMAT-value=42")
+
     def test_raw_tstrings(self):
         path = r"C:\Users"
         t = rt"{path}\Documents"

Parser/action_helpers.c

@@ -1001,14 +1001,21 @@ result_token_with_metadata(Parser *p, void *result, PyObject *metadata)
     return res;
 }
 
+static char
+formatted_string_prefix(const Parser *p)
+{
+    const ftstring_state *state = _PyLexer_CurrentFTString(p->tok);
+    return state == NULL ? 'f' : _PyLexer_StringPrefix(state->kind);
+}
+
 ResultTokenWithMetadata *
 _PyPegen_check_fstring_conversion(Parser *p, Token* conv_token, expr_ty conv)
 {
     if (conv_token->lineno != conv->lineno || conv_token->end_col_offset != conv->col_offset) {
         return RAISE_SYNTAX_ERROR_KNOWN_RANGE(
             conv_token, conv,
             "%c-string: conversion type must come right after the exclamation mark",
-            TOK_GET_STRING_PREFIX(p->tok)
+            formatted_string_prefix(p)
         );
     }
 
@@ -1017,7 +1024,7 @@ _PyPegen_check_fstring_conversion(Parser *p, Token* conv_token, expr_ty conv)
             !(first == 's' || first == 'r' || first == 'a')) {
         RAISE_SYNTAX_ERROR_KNOWN_LOCATION(conv,
                                             "%c-string: invalid conversion character %R: expected 's', 'r', or 'a'",
-                                            TOK_GET_STRING_PREFIX(p->tok),
+                                            formatted_string_prefix(p),
                                             conv->v.Name.id);
         return NULL;
     }
@@ -1344,7 +1351,8 @@ _PyPegen_decode_fstring_part(Parser* p, int is_raw, expr_ty constant, Token* tok
 }
 
 static asdl_expr_seq *
-_get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Token *b, enum string_kind_t string_kind)
+_get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions,
+                   Token *b, ftstring_kind string_kind)
 {
     Py_ssize_t n_items = asdl_seq_LEN(raw_expressions);
     Py_ssize_t total_items = n_items;
@@ -1370,15 +1378,13 @@ _get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Token *b
     for (Py_ssize_t i = 0; i < n_items; i++) {
         expr_ty item = asdl_seq_GET(raw_expressions, i);
 
-        // This should correspond to a JoinedStr node of two elements
-        // created _PyPegen_formatted_value. This situation can only be the result of
-        // a (f|t)-string debug expression where the first element is a constant with the text and the second
-        // a formatted value with the expression.
+        /* Debug expressions arrive as JoinedStr(text, value); flatten them
+           into the surrounding string. */
         if (item->kind == JoinedStr_kind) {
             asdl_expr_seq *values = item->v.JoinedStr.values;
             if (asdl_seq_LEN(values) != 2) {
                 PyErr_Format(PyExc_SystemError,
-                             string_kind == TSTRING
+                             _PyLexer_IsTString(string_kind)
                              ? "unexpected TemplateStr node without debug data in t-string at line %d"
                              : "unexpected JoinedStr node without debug data in f-string at line %d",
                              item->lineno);
@@ -1390,7 +1396,9 @@ _get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Token *b
             asdl_seq_SET(seq, index++, first);
 
             expr_ty second = asdl_seq_GET(values, 1);
-            assert((string_kind == TSTRING && second->kind == Interpolation_kind) || second->kind == FormattedValue_kind);
+            assert((_PyLexer_IsTString(string_kind) &&
+                    second->kind == Interpolation_kind) ||
+                   second->kind == FormattedValue_kind);
             asdl_seq_SET(seq, index++, second);
 
             continue;
@@ -1460,12 +1468,8 @@ expr_ty _PyPegen_decoded_constant_from_token(Parser* p, Token* tok) {
         return NULL;
     }
 
-    // Check if we're inside a raw f-string for format spec decoding
-    int is_raw = 0;
-    if (INSIDE_FSTRING(p->tok)) {
-        tokenizer_mode *mode = TOK_GET_MODE(p->tok);
-        is_raw = mode->raw;
-    }
+    const ftstring_state *state = _PyLexer_CurrentFTString(p->tok);
+    int is_raw = state != NULL && _PyLexer_IsRawString(state->kind);
 
     PyObject* str = _PyPegen_decode_string(p, is_raw, bstr, bsize, tok);
     if (str == NULL) {

Parser/lexer/lexer.c

@@ -154,8 +154,11 @@ tok_continuation_line(struct tok_state *tok) {
 
 
 int
-_PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct token *token)
+_PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token *token)
 {
+    assert(current == NULL ||
+           (current->mode == FTSTRING_MODE_EXPRESSION &&
+            current->replacement_depth > 0));
     int c;
     int blankline, nonascii;
 
@@ -317,13 +320,13 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
             c = tok_nextc(tok);
         }
 
-        if (INSIDE_FSTRING(tok) && INSIDE_FSTRING_EXPR(current_tok)) {
+        if (current != NULL) {
             const char *comment_end = tok->cur;
             if (c == '\n' || c == '\r') {
                 comment_end--;
             }
             if (_PyLexer_record_ftstring_comment(
-                    tok, tok->start, comment_end) < 0) {
+                    tok, current, tok->start, comment_end) < 0) {
                 tok->done = E_NOMEM;
                 return MAKE_TOKEN(ERRORTOKEN);
             }
@@ -547,34 +550,25 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
     }
 
     /* Punctuation character */
-    int is_punctuation = (c == ':' || c == '}' || c == '!' || c == '{');
-    if (is_punctuation && INSIDE_FSTRING(tok) && INSIDE_FSTRING_EXPR(current_tok)) {
-        /* This code block gets executed before the curly_bracket_depth is incremented
-         * by the `{` case, so for ensuring that we are on the 0th level, we need
-         * to adjust it manually */
-        int cursor = current_tok->curly_bracket_depth - (c != '{');
-        int in_format_spec = current_tok->in_format_spec;
-         int cursor_in_format_with_debug =
-             cursor == 1 && (current_tok->in_debug || in_format_spec);
-         int cursor_valid = cursor == 0 || cursor_in_format_with_debug;
-        if (cursor_valid && c == '!') {
+    int is_punctuation = (c == ':' || c == '}' || c == '!');
+    if (is_punctuation && current != NULL) {
+        int bracket_depth = _PyLexer_FTStringBracketDepth(tok, current);
+        int at_expression_boundary =
+            bracket_depth == current->replacement_depth;
+        if (at_expression_boundary && c == '!') {
             int c2 = tok_nextc(tok);
             if (c2 == '=') {
-                cursor_valid = 0;
+                at_expression_boundary = 0;
             }
             tok_backup(tok, c2);
         }
-        if (cursor_valid) {
-            _PyLexer_update_ftstring_expr(tok, c);
-        }
-        if (cursor_valid && c != '{' &&
-                _PyLexer_set_ftstring_expr_metadata(tok, token)) {
+        if (at_expression_boundary &&
+                _PyLexer_finish_ftstring_expr(tok, current, token)) {
             return MAKE_TOKEN(ERRORTOKEN);
         }
 
-        if (c == ':' && cursor == current_tok->curly_bracket_expr_start_depth) {
-            current_tok->kind = TOK_FSTRING_MODE;
-            current_tok->in_format_spec = 1;
+        if (c == ':' && at_expression_boundary) {
+            current->mode = FTSTRING_MODE_FORMAT_SPEC;
             p_start = tok->start;
             p_end = tok->cur;
             return MAKE_TOKEN(_PyToken_OneChar(c));
@@ -613,16 +607,20 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
         tok->parenlinenostack[tok->level] = tok->lineno;
         tok->parencolstack[tok->level] = (int)(tok->start - tok->line_start);
         tok->level++;
-        if (INSIDE_FSTRING(tok)) {
-            current_tok->curly_bracket_depth++;
-        }
         break;
     case ')':
     case ']':
     case '}':
-        if (INSIDE_FSTRING(tok) && !current_tok->curly_bracket_depth && c == '}') {
-            return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
-                "%c-string: single '}' is not allowed", TOK_GET_STRING_PREFIX(tok)));
+        if (current != NULL &&
+                _PyLexer_FTStringBracketDepth(tok, current) == 0) {
+            if (c == '}') {
+                return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
+                    "%c-string: single '}' is not allowed",
+                    _PyLexer_StringPrefix(current->kind)));
+            }
+            return MAKE_TOKEN(_PyTokenizer_syntaxerror(
+                tok, "%c-string: unmatched '%c'",
+                _PyLexer_StringPrefix(current->kind), c));
         }
         if (!tok->tok_extra_tokens && !tok->level) {
             return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "unmatched '%c'", c));
@@ -633,17 +631,15 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
             if (!tok->tok_extra_tokens && !((opening == '(' && c == ')') ||
                                             (opening == '[' && c == ']') ||
                                             (opening == '{' && c == '}'))) {
-                /* If the opening bracket belongs to an f-string's expression
-                part (e.g. f"{)}") and the closing bracket is an arbitrary
-                nested expression, then instead of matching a different
-                syntactical construct with it; we'll throw an unmatched
-                parentheses error. */
-                if (INSIDE_FSTRING(tok) && opening == '{') {
-                    assert(current_tok->curly_bracket_depth >= 0);
-                    int previous_bracket = current_tok->curly_bracket_depth - 1;
-                    if (previous_bracket == current_tok->curly_bracket_expr_start_depth) {
+                /* Do not match a closer against the brace that opened the
+                 * current replacement field. */
+                if (current != NULL && opening == '{') {
+                    int bracket_depth =
+                        _PyLexer_FTStringBracketDepth(tok, current);
+                    if (bracket_depth == current->replacement_depth - 1) {
                         return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
-                            "%c-string: unmatched '%c'", TOK_GET_STRING_PREFIX(tok), c));
+                            "%c-string: unmatched '%c'",
+                            _PyLexer_StringPrefix(current->kind), c));
                     }
                 }
                 if (tok->parenlinenostack[tok->level] != tok->lineno) {
@@ -661,17 +657,16 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
             }
         }
 
-        if (INSIDE_FSTRING(tok)) {
-            current_tok->curly_bracket_depth--;
-            if (current_tok->curly_bracket_depth < 0) {
+        if (current != NULL) {
+            int bracket_depth = _PyLexer_FTStringBracketDepth(tok, current);
+            if (bracket_depth < 0) {
                 return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "%c-string: unmatched '%c'",
-                    TOK_GET_STRING_PREFIX(tok), c));
+                    _PyLexer_StringPrefix(current->kind), c));
             }
-            if (c == '}' && current_tok->curly_bracket_depth == current_tok->curly_bracket_expr_start_depth) {
-                current_tok->curly_bracket_expr_start_depth--;
-                current_tok->kind = TOK_FSTRING_MODE;
-                current_tok->in_format_spec = 0;
-                current_tok->in_debug = 0;
+            if (c == '}' && bracket_depth == current->replacement_depth - 1) {
+                current->replacement_depth--;
+                current->mode = FTSTRING_MODE_MIDDLE;
+                current->debug_expr = 0;
             }
         }
         break;
@@ -683,8 +678,9 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
         return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid non-printable character U+%04X", c));
     }
 
-    if( c == '=' && INSIDE_FSTRING_EXPR_AT_TOP(current_tok)) {
-        current_tok->in_debug = 1;
+    if (c == '=' && current != NULL &&
+            _PyLexer_FTStringBracketDepth(tok, current) == current->replacement_depth) {
+        current->debug_expr = 1;
     }
 
     /* Punctuation character */
@@ -694,21 +690,13 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
 }
 
 
-static int
-tok_get(struct tok_state *tok, struct token *token)
-{
-    tokenizer_mode *current_tok = TOK_GET_MODE(tok);
-    if (current_tok->kind == TOK_REGULAR_MODE) {
-        return _PyLexer_get_normal_mode(tok, current_tok, token);
-    } else {
-        return _PyLexer_get_fstring_mode(tok, current_tok, token);
-    }
-}
-
 int
 _PyTokenizer_Get(struct tok_state *tok, struct token *token)
 {
-    int result = tok_get(tok, token);
+    ftstring_state *current = _PyLexer_CurrentFTString(tok);
+    int result = current == NULL || current->mode == FTSTRING_MODE_EXPRESSION
+        ? _PyLexer_get_normal(tok, current, token)
+        : _PyLexer_get_ftstring(tok, current, token);
     if (tok_failed(tok)) {
         result = ERRORTOKEN;
     }

Parser/lexer/lexer_internal.h

@@ -17,31 +17,10 @@
                || c == '_'\
                || (c >= 128))
 
-#ifdef Py_DEBUG
-static inline tokenizer_mode *
-TOK_GET_MODE(struct tok_state *tok)
-{
-    assert(tok->tok_mode_stack_index >= 0);
-    assert(tok->tok_mode_stack_index < MAXFSTRINGLEVEL);
-    return &tok->tok_mode_stack[tok->tok_mode_stack_index];
-}
-
-static inline tokenizer_mode *
-TOK_NEXT_MODE(struct tok_state *tok)
-{
-    assert(tok->tok_mode_stack_index >= 0);
-    assert(tok->tok_mode_stack_index + 1 < MAXFSTRINGLEVEL);
-    return &tok->tok_mode_stack[++tok->tok_mode_stack_index];
-}
-#else
-#define TOK_GET_MODE(tok) (&(tok)->tok_mode_stack[(tok)->tok_mode_stack_index])
-#define TOK_NEXT_MODE(tok) (&(tok)->tok_mode_stack[++(tok)->tok_mode_stack_index])
-#endif
-
-#define FTSTRING_MIDDLE(tok_mode) ((tok_mode)->string_kind == TSTRING ? TSTRING_MIDDLE : FSTRING_MIDDLE)
-#define FTSTRING_END(tok_mode) ((tok_mode)->string_kind == TSTRING ? TSTRING_END : FSTRING_END)
-#define TOK_GET_STRING_PREFIX(tok) (TOK_GET_MODE(tok)->string_kind == TSTRING ? 't' : 'f')
-
+#define FTSTRING_MIDDLE(state) \
+    (_PyLexer_IsTString((state)->kind) ? TSTRING_MIDDLE : FSTRING_MIDDLE)
+#define FTSTRING_END(state) \
+    (_PyLexer_IsTString((state)->kind) ? TSTRING_END : FSTRING_END)
 #define tok_nextc _PyLexer_nextc
 #define tok_backup _PyLexer_backup
 
@@ -54,15 +33,15 @@ tok_failed(const struct tok_state *tok)
 
 int _PyLexer_nextc(struct tok_state *);
 void _PyLexer_backup(struct tok_state *, int);
-void _PyLexer_update_ftstring_expr(struct tok_state *, char);
 int _PyLexer_record_ftstring_comment(
-    struct tok_state *, const char *, const char *);
-int _PyLexer_set_ftstring_expr_metadata(struct tok_state *, struct token *);
+    struct tok_state *, ftstring_state *, const char *, const char *);
+int _PyLexer_finish_ftstring_expr(
+    struct tok_state *, ftstring_state *, struct token *);
 int _PyLexer_check_string_prefixes(struct tok_state *, int, int, int, int, int);
 int _PyLexer_scan_number(struct tok_state *, struct token *, int, int);
 int _PyLexer_scan_fstring_start(struct tok_state *, struct token *, int);
 int _PyLexer_scan_string(struct tok_state *, struct token *, int);
-int _PyLexer_get_normal_mode(struct tok_state *, tokenizer_mode *, struct token *);
-int _PyLexer_get_fstring_mode(struct tok_state *, tokenizer_mode *, struct token *);
+int _PyLexer_get_normal(struct tok_state *, ftstring_state *, struct token *);
+int _PyLexer_get_ftstring(struct tok_state *, ftstring_state *, struct token *);
 
 #endif