Skip to content

Commit c0ee295

Browse files
pablogsalpablogsal
committed
gh-153569: centralize formatted-string transitions
1 parent 9e645ae commit c0ee295

3 files changed

Lines changed: 112 additions & 51 deletions

File tree

‎Parser/lexer/lexer.c‎

Lines changed: 26 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -552,26 +552,14 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token
552552
/* Punctuation character */
553553
int is_punctuation = (c == ':' || c == '}' || c == '!');
554554
if (is_punctuation && current != NULL) {
555-
int bracket_depth = _PyLexer_FTStringBracketDepth(tok, current);
556-
int at_expression_boundary =
557-
bracket_depth == current->replacement_depth;
558-
if (at_expression_boundary && c == '!') {
559-
int c2 = tok_nextc(tok);
560-
if (c2 == '=') {
561-
at_expression_boundary = 0;
562-
}
563-
tok_backup(tok, c2);
564-
}
565-
if (at_expression_boundary &&
566-
_PyLexer_finish_ftstring_expr(tok, current, token)) {
555+
int type = _PyLexer_ftstring_punctuation(tok, current, token, c);
556+
if (type < 0) {
567557
return MAKE_TOKEN(ERRORTOKEN);
568558
}
569-
570-
if (c == ':' && at_expression_boundary) {
571-
current->mode = FTSTRING_MODE_FORMAT_SPEC;
559+
if (type != 0) {
572560
p_start = tok->start;
573561
p_end = tok->cur;
574-
return MAKE_TOKEN(_PyToken_OneChar(c));
562+
return MAKE_TOKEN(type);
575563
}
576564
}
577565

@@ -656,18 +644,9 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token
656644
}
657645
}
658646
}
659-
660-
if (current != NULL) {
661-
int bracket_depth = _PyLexer_FTStringBracketDepth(tok, current);
662-
if (bracket_depth < 0) {
663-
return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "%c-string: unmatched '%c'",
664-
_PyLexer_StringPrefix(current->kind), c));
665-
}
666-
if (c == '}' && bracket_depth == current->replacement_depth - 1) {
667-
current->replacement_depth--;
668-
current->mode = FTSTRING_MODE_MIDDLE;
669-
current->debug_expr = 0;
670-
}
647+
if (current != NULL &&
648+
_PyLexer_close_ftstring_expr(tok, current, c) < 0) {
649+
return MAKE_TOKEN(ERRORTOKEN);
671650
}
672651
break;
673652
default:
@@ -678,9 +657,8 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token
678657
return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid non-printable character U+%04X", c));
679658
}
680659

681-
if (c == '=' && current != NULL &&
682-
_PyLexer_FTStringBracketDepth(tok, current) == current->replacement_depth) {
683-
current->debug_expr = 1;
660+
if (c == '=' && current != NULL) {
661+
_PyLexer_mark_ftstring_debug(tok, current);
684662
}
685663

686664
/* Punctuation character */
@@ -694,9 +672,23 @@ int
694672
_PyTokenizer_Get(struct tok_state *tok, struct token *token)
695673
{
696674
ftstring_state *current = _PyLexer_CurrentFTString(tok);
697-
int result = current == NULL || current->mode == FTSTRING_MODE_EXPRESSION
698-
? _PyLexer_get_normal(tok, current, token)
699-
: _PyLexer_get_ftstring(tok, current, token);
675+
int result;
676+
if (current == NULL) {
677+
result = _PyLexer_get_normal(tok, NULL, token);
678+
}
679+
else {
680+
switch (current->mode) {
681+
case FTSTRING_MODE_EXPRESSION:
682+
result = _PyLexer_get_normal(tok, current, token);
683+
break;
684+
case FTSTRING_MODE_MIDDLE:
685+
case FTSTRING_MODE_FORMAT_SPEC:
686+
result = _PyLexer_get_ftstring(tok, current, token);
687+
break;
688+
default:
689+
Py_UNREACHABLE();
690+
}
691+
}
700692
if (tok_failed(tok)) {
701693
result = ERRORTOKEN;
702694
}

‎Parser/lexer/lexer_internal.h‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,8 +35,11 @@ int _PyLexer_nextc(struct tok_state *);
3535
void _PyLexer_backup(struct tok_state *, int);
3636
int _PyLexer_record_ftstring_comment(
3737
struct tok_state *, ftstring_state *, const char *, const char *);
38-
int _PyLexer_finish_ftstring_expr(
39-
struct tok_state *, ftstring_state *, struct token *);
38+
int _PyLexer_ftstring_punctuation(
39+
struct tok_state *, ftstring_state *, struct token *, int);
40+
int _PyLexer_close_ftstring_expr(
41+
struct tok_state *, ftstring_state *, int);
42+
void _PyLexer_mark_ftstring_debug(struct tok_state *, ftstring_state *);
4043
int _PyLexer_check_string_prefixes(struct tok_state *, int, int, int, int, int);
4144
int _PyLexer_scan_number(struct tok_state *, struct token *, int, int);
4245
int _PyLexer_scan_fstring_start(struct tok_state *, struct token *, int);

‎Parser/lexer/string.c‎

Lines changed: 81 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -54,9 +54,9 @@ _PyLexer_record_ftstring_comment(struct tok_state *tok, ftstring_state *state,
5454
return 0;
5555
}
5656

57-
int
58-
_PyLexer_finish_ftstring_expr(struct tok_state *tok, ftstring_state *state,
59-
struct token *token)
57+
static int
58+
finish_ftstring_expr(struct tok_state *tok, ftstring_state *state,
59+
struct token *token)
6060
{
6161
assert(token != NULL && state == _PyLexer_CurrentFTString(tok));
6262
assert(state->mode == FTSTRING_MODE_EXPRESSION && tok->start != NULL);
@@ -127,6 +127,82 @@ _PyLexer_finish_ftstring_expr(struct tok_state *tok, ftstring_state *state,
127127
return 0;
128128
}
129129

130+
static int
131+
begin_ftstring_expr(struct tok_state *tok, ftstring_state *state,
132+
_PyTok_Off expr_start)
133+
{
134+
assert(state->mode != FTSTRING_MODE_EXPRESSION);
135+
state->expr_span = (_PyTok_Span){expr_start, -1};
136+
if (state->comments != NULL) {
137+
state->comments->count = 0;
138+
}
139+
if (state->replacement_depth >= MAX_EXPR_NESTING) {
140+
_PyTokenizer_syntaxerror(
141+
tok, "%c-string: expressions nested too deeply",
142+
_PyLexer_StringPrefix(state->kind));
143+
return -1;
144+
}
145+
state->replacement_depth++;
146+
state->mode = FTSTRING_MODE_EXPRESSION;
147+
state->debug_expr = 0;
148+
return 0;
149+
}
150+
151+
int
152+
_PyLexer_ftstring_punctuation(struct tok_state *tok, ftstring_state *state,
153+
struct token *token, int c)
154+
{
155+
assert(state->mode == FTSTRING_MODE_EXPRESSION);
156+
assert(c == ':' || c == '}' || c == '!');
157+
if (_PyLexer_FTStringBracketDepth(tok, state) != state->replacement_depth) {
158+
return 0;
159+
}
160+
if (c == '!') {
161+
int next = tok_nextc(tok);
162+
tok_backup(tok, next);
163+
if (next == '=') {
164+
return 0;
165+
}
166+
}
167+
if (finish_ftstring_expr(tok, state, token) < 0) {
168+
return -1;
169+
}
170+
if (c == ':') {
171+
state->mode = FTSTRING_MODE_FORMAT_SPEC;
172+
return COLON;
173+
}
174+
return 0;
175+
}
176+
177+
int
178+
_PyLexer_close_ftstring_expr(struct tok_state *tok, ftstring_state *state,
179+
int c)
180+
{
181+
assert(state->mode == FTSTRING_MODE_EXPRESSION);
182+
assert(c == ')' || c == ']' || c == '}');
183+
int depth = _PyLexer_FTStringBracketDepth(tok, state);
184+
if (depth < 0) {
185+
_PyTokenizer_syntaxerror(tok, "%c-string: unmatched '%c'",
186+
_PyLexer_StringPrefix(state->kind), c);
187+
return -1;
188+
}
189+
if (c == '}' && depth == state->replacement_depth - 1) {
190+
state->replacement_depth--;
191+
state->mode = FTSTRING_MODE_MIDDLE;
192+
state->debug_expr = 0;
193+
}
194+
return 0;
195+
}
196+
197+
void
198+
_PyLexer_mark_ftstring_debug(struct tok_state *tok, ftstring_state *state)
199+
{
200+
assert(state->mode == FTSTRING_MODE_EXPRESSION);
201+
if (_PyLexer_FTStringBracketDepth(tok, state) == state->replacement_depth) {
202+
state->debug_expr = 1;
203+
}
204+
}
205+
130206
int
131207
_PyLexer_check_string_prefixes(struct tok_state *tok,
132208
int saw_b, int saw_r, int saw_u,
@@ -408,21 +484,11 @@ _PyLexer_get_ftstring(struct tok_state *tok, ftstring_state *current, struct tok
408484
int peek = tok_nextc(tok);
409485
if (peek != '{' || in_format_spec) {
410486
tok_backup(tok, peek);
411-
current->expr_span = (_PyTok_Span){
412-
_PyLexer_BufferOffset(tok, tok->cur), -1};
413-
if (current->comments != NULL) {
414-
current->comments->count = 0;
415-
}
487+
_PyTok_Off expr_start = _PyLexer_BufferOffset(tok, tok->cur);
416488
tok_backup(tok, c);
417-
if (current->replacement_depth >= MAX_EXPR_NESTING) {
418-
_PyTokenizer_syntaxerror(
419-
tok, "%c-string: expressions nested too deeply",
420-
_PyLexer_StringPrefix(current->kind));
489+
if (begin_ftstring_expr(tok, current, expr_start) < 0) {
421490
return MAKE_TOKEN(ERRORTOKEN);
422491
}
423-
current->replacement_depth++;
424-
current->mode = FTSTRING_MODE_EXPRESSION;
425-
current->debug_expr = 0;
426492
p_start = tok->start;
427493
p_end = tok->cur;
428494
if (p_start == p_end) {

0 commit comments

Comments
 (0)