gh-153569: Split tokenizer lexer into focused files

Lib/test/test_fstring.py

@@ -709,6 +709,16 @@ def test_double_braces(self):
                             ["f'{ {{}} }'", # dict in a set
                              ])
 
+    def test_double_brace_ast_location_covers_both_source_braces(self):
+        value = ast.parse('f"a{{"').body[0].value.values[0]
+        self.assertIsInstance(value, ast.Constant)
+        self.assertEqual(value.value, "a{")
+        self.assertEqual(
+            (value.lineno, value.col_offset, value.end_lineno,
+             value.end_col_offset),
+            (1, 2, 1, 5),
+        )
+
     def test_compile_time_concat(self):
         x = 'def'
         self.assertEqual('abc' f'## {x}ghi', 'abc## defghi')

Lib/test/test_tokenize.py

@@ -2236,6 +2236,13 @@ def test_number_starting_with_zero(self):
         self.assertEqual(tokens, expected_tokens)
 
 class CTokenizeTest(TestCase):
+    @staticmethod
+    def _get_tokens(source, *, extra_tokens=False):
+        return list(tokenize._generate_tokens_from_c_tokenizer(
+            StringIO(source).readline,
+            extra_tokens=extra_tokens,
+        ))
+
     def check_tokenize(self, s, expected):
         # Format the tokens in s in a table format.
         # The ENDMARKER and final NEWLINE are omitted.
@@ -2266,6 +2273,159 @@ def readline(encoding):
                 ))
                 self.assertEqual(tokens, expected)
 
+    def test_extra_tokens_relaxes_lexer_errors(self):
+        cases = [
+            (
+                "2sin(x)",
+                ("invalid decimal literal", (1, 1)),
+                [
+                    (token.NUMBER, "2", (1, 0), (1, 1)),
+                    (token.NAME, "sin", (1, 1), (1, 4)),
+                    (token.OP, "(", (1, 4), (1, 5)),
+                    (token.NAME, "x", (1, 5), (1, 6)),
+                    (token.OP, ")", (1, 6), (1, 7)),
+                ],
+            ),
+            (
+                "01234",
+                (
+                    "leading zeros in decimal integer literals are not permitted; "
+                    "use an 0o prefix for octal integers",
+                    (1, 1),
+                ),
+                [(token.NUMBER, "01234", (1, 0), (1, 5))],
+            ),
+            (
+                ")",
+                ("unmatched ')'", (1, 1)),
+                [(token.OP, ")", (1, 0), (1, 1))],
+            ),
+            (
+                "(]",
+                (
+                    "closing parenthesis ']' does not match opening parenthesis '('",
+                    (1, 2),
+                ),
+                [
+                    (token.OP, "(", (1, 0), (1, 1)),
+                    (token.OP, "]", (1, 1), (1, 2)),
+                ],
+            ),
+            (
+                "a☃b",
+                ("invalid character '☃' (U+2603)", (1, 2)),
+                [(token.NAME, "a☃b", (1, 0), (1, 3))],
+            ),
+        ]
+
+        for source, error, expected in cases:
+            with self.subTest(source=source):
+                with self.assertRaises(tokenize.TokenError) as caught:
+                    self._get_tokens(source)
+                self.assertEqual(caught.exception.args, error)
+
+                tokens = self._get_tokens(source, extra_tokens=True)
+                self.assertEqual(
+                    [(tok.type, tok.string, tok.start, tok.end)
+                     for tok in tokens[:-2]],
+                    expected,
+                )
+
+    def test_extra_tokens_emits_comments_and_real_indent_locations(self):
+        source = "if x:\n  # c\n  y\nz\n"
+        kinds = {token.COMMENT, token.NL, token.INDENT, token.DEDENT}
+
+        parser_tokens = self._get_tokens(source)
+        self.assertEqual(
+            [(tok.type, tok.string, tok.start, tok.end)
+             for tok in parser_tokens if tok.type in kinds],
+            [
+                (token.INDENT, "", (3, -1), (3, -1)),
+                (token.DEDENT, "", (4, -1), (4, -1)),
+            ],
+        )
+
+        tolerant_tokens = self._get_tokens(source, extra_tokens=True)
+        self.assertEqual(
+            [(tok.type, tok.string, tok.start, tok.end)
+             for tok in tolerant_tokens if tok.type in kinds],
+            [
+                (token.COMMENT, "# c", (2, 2), (2, 5)),
+                (token.NL, "\n", (2, 5), (2, 6)),
+                (token.INDENT, "  ", (3, 0), (3, 2)),
+                (token.DEDENT, "", (4, 0), (4, 0)),
+            ],
+        )
+
+    def test_printable_invalid_operator_streams_in_both_modes(self):
+        for extra_tokens in (False, True):
+            with self.subTest(extra_tokens=extra_tokens):
+                first = self._get_tokens(
+                    "1 $ 2",
+                    extra_tokens=extra_tokens,
+                )[1]
+                self.assertEqual(
+                    (first.type, first.string, first.start, first.end),
+                    (token.OP, "$", (1, 2), (1, 3)),
+                )
+
+    def test_degraded_fstring_format_spec(self):
+        tokens = self._get_tokens('f"{1:{2}{{3}}}"')
+        self.assertEqual(
+            [(tok.string, tok.start, tok.end)
+             for tok in tokens if tok.type == token.FSTRING_MIDDLE],
+            [
+                ("{", (1, 8), (1, 9)),
+                ("3", (1, 10), (1, 11)),
+                ("}", (1, 12), (1, 13)),
+            ],
+        )
+
+        tokens = self._get_tokens('f"{x:{y}}"')
+        self.assertEqual(
+            [(tok.string, tok.start, tok.end)
+             for tok in tokens if tok.type == token.FSTRING_MIDDLE],
+            [("", (1, 8), (1, 8))],
+        )
+
+        with self.assertRaises(tokenize.TokenError) as caught:
+            self._get_tokens('f"{1:{2}x}}y"')
+        self.assertEqual(
+            caught.exception.args,
+            ("f-string: single '}' is not allowed", (1, 11)),
+        )
+
+    def test_escaped_fstring_brace_has_a_position_gap(self):
+        tokens = self._get_tokens('f"a{{"', extra_tokens=True)
+        self.assertEqual(
+            [(tok.type, tok.string, tok.start, tok.end)
+             for tok in tokens
+             if tok.type in {token.FSTRING_MIDDLE, token.FSTRING_END}],
+            [
+                (token.FSTRING_MIDDLE, "a{", (1, 2), (1, 4)),
+                (token.FSTRING_END, '"', (1, 5), (1, 6)),
+            ],
+        )
+
+    def test_tolerant_incompatible_prefix_position_after_non_ascii(self):
+        with self.assertRaises(tokenize.TokenError) as caught:
+            self._get_tokens('bé )tf"2 ', extra_tokens=True)
+        self.assertEqual(
+            caught.exception.args,
+            ("'f' and 't' prefixes are incompatible", (1, 6)),
+        )
+
+    def test_tolerant_fstring_closer_at_expression_entry_depth(self):
+        source = "f'1:{]]{}}r}''"
+        for extra_tokens, position in ((False, (1, 6)), (True, (1, 7))):
+            with self.subTest(extra_tokens=extra_tokens):
+                with self.assertRaises(tokenize.TokenError) as caught:
+                    self._get_tokens(source, extra_tokens=extra_tokens)
+                self.assertEqual(
+                    caught.exception.args,
+                    ("f-string: unmatched ']'", position),
+                )
+
     def test_int(self):
 
         self.check_tokenize('0xff <= 255', """\

Makefile.pre.in

@@ -395,7 +395,9 @@ PEGEN_OBJS=		\
 TOKENIZER_OBJS=		\
 		Parser/lexer/buffer.o \
 		Parser/lexer/lexer.o \
+		Parser/lexer/number.o \
 		Parser/lexer/state.o \
+		Parser/lexer/string.o \
 		Parser/tokenizer/file_tokenizer.o \
 		Parser/tokenizer/readline_tokenizer.o \
 		Parser/tokenizer/string_tokenizer.o \
@@ -410,6 +412,7 @@ PEGEN_HEADERS= \
 TOKENIZER_HEADERS= \
 		Parser/lexer/buffer.h \
 		Parser/lexer/lexer.h \
+		Parser/lexer/lexer_internal.h \
 		Parser/lexer/state.h \
 		Parser/tokenizer/tokenizer.h \
 		Parser/tokenizer/helpers.h

PCbuild/_freeze_module.vcxproj

@@ -184,6 +184,8 @@
     <ClCompile Include="..\Parser\lexer\buffer.c" />
     <ClCompile Include="..\Parser\lexer\state.c" />
     <ClCompile Include="..\Parser\lexer\lexer.c" />
+    <ClCompile Include="..\Parser\lexer\number.c" />
+    <ClCompile Include="..\Parser\lexer\string.c" />
     <ClCompile Include="..\Parser\tokenizer\string_tokenizer.c" />
     <ClCompile Include="..\Parser\tokenizer\file_tokenizer.c" />
     <ClCompile Include="..\Parser\tokenizer\utf8_tokenizer.c" />

PCbuild/_freeze_module.vcxproj.filters

@@ -463,6 +463,12 @@
     <ClCompile Include="..\Parser\lexer\lexer.c">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\Parser\lexer\number.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Parser\lexer\string.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="..\Parser\lexer\buffer.c">
       <Filter>Source Files</Filter>
     </ClCompile>

PCbuild/pythoncore.vcxproj

@@ -420,6 +420,7 @@
     <ClInclude Include="..\Objects\unicodetype_db.h" />
     <ClInclude Include="..\Parser\lexer\state.h" />
     <ClInclude Include="..\Parser\lexer\lexer.h" />
+    <ClInclude Include="..\Parser\lexer\lexer_internal.h" />
     <ClInclude Include="..\Parser\lexer\buffer.h" />
     <ClInclude Include="..\Parser\tokenizer\helpers.h" />
     <ClInclude Include="..\Parser\tokenizer\tokenizer.h" />
@@ -584,6 +585,8 @@
     <ClCompile Include="..\Parser\myreadline.c" />
     <ClCompile Include="..\Parser\lexer\state.c" />
     <ClCompile Include="..\Parser\lexer\lexer.c" />
+    <ClCompile Include="..\Parser\lexer\number.c" />
+    <ClCompile Include="..\Parser\lexer\string.c" />
     <ClCompile Include="..\Parser\lexer\buffer.c" />
     <ClCompile Include="..\Parser\tokenizer\string_tokenizer.c" />
     <ClCompile Include="..\Parser\tokenizer\file_tokenizer.c" />

PCbuild/pythoncore.vcxproj.filters

@@ -327,6 +327,9 @@
     <ClInclude Include="..\Parser\lexer\state.h">
       <Filter>Parser</Filter>
     </ClInclude>
+    <ClInclude Include="..\Parser\lexer\lexer_internal.h">
+      <Filter>Parser</Filter>
+    </ClInclude>
     <ClInclude Include="..\Parser\lexer\buffer.h">
       <Filter>Parser</Filter>
     </ClInclude>
@@ -1331,6 +1334,12 @@
     <ClCompile Include="..\Parser\lexer\lexer.c">
       <Filter>Parser</Filter>
     </ClCompile>
+    <ClCompile Include="..\Parser\lexer\number.c">
+      <Filter>Parser</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Parser\lexer\string.c">
+      <Filter>Parser</Filter>
+    </ClCompile>
     <ClCompile Include="..\Parser\lexer\state.c">
       <Filter>Parser</Filter>
     </ClCompile>