gh-153569: finish tokenizer offsets and the opaque API cutover

Use persistent decoded-source offsets for scanner positions and explicit
reporting locations and text for diagnostics, including incomplete input.

Move pegen and _tokenize behind tokenizer.h operations and remove their
dependence on lexer layout. Remove the obsolete cursor, source lookup,
and compatibility APIs after the consumer cutover.

Author: pablogsal

Lib/test/test_capi/test_tokenizer.py

@@ -12,9 +12,6 @@ def test_source(self):
     def test_source_discard(self):
         _testinternalcapi.test_tokenizer_source_discard()
 
-    def test_cursor(self):
-        _testinternalcapi.test_tokenizer_cursor()
-
 
 if __name__ == "__main__":
     unittest.main()

Lib/test/test_codeop.py

@@ -113,6 +113,17 @@ def test_valid(self, compiler):
         av("def f():\n pass\n#foo\n")
         av("@a.b.c\ndef f():\n pass\n")
 
+    @subTests('symbol', ('single', 'exec'))
+    @subTests('prefix', ('', 'f', 't'))
+    def test_incomplete_string_diagnostics(self, symbol, prefix):
+        opening = f'    á = {prefix}"""first\n'
+        source = 'if True:\n' + opening + 'second'
+        with self.assertRaises(_IncompleteInputError) as cm:
+            Compile()(source, '<input>', symbol)
+        text = opening + 'second' + ('\n' if symbol == 'exec' else '')
+        self.assertEqual(cm.exception.args, (
+            'incomplete input', ('<input>', 2, 9, text, 2, -1)))
+
     @subTests('compiler', COMPILERS)
     def test_incomplete(self, compiler):
         ai = functools.partial(self.assertIncomplete, compiler=compiler)

Lib/test/test_source_encoding.py

@@ -3,8 +3,8 @@
 import unittest
 from test import support
 from test.support import script_helper
-from test.support.os_helper import TESTFN, unlink, rmtree
-from test.support.import_helper import unload
+from test.support.os_helper import TESTFN, TESTFN_ASCII, unlink, rmtree
+from test.support.import_helper import import_module, unload
 import importlib
 import os
 import sys
@@ -83,12 +83,30 @@ def test_truncated_utf8_at_eof(self):
                 self.assertRaises(SyntaxError, compile, seq, '<test>', 'exec')
 
     def test_invalid_utf8_offset_after_non_ascii(self):
+        for name in ('é', 'éé', '𝒜'):
+            with self.subTest(name=name):
+                source = ('x = ' + name).encode() + b'\xff\n'
+                with self.assertRaises(SyntaxError) as caught:
+                    compile(source, '<test>', 'exec')
+                error = caught.exception
+                self.assertEqual(
+                    (error.lineno, error.offset, error.end_lineno, error.end_offset),
+                    (1, 5 + len(name), 1, 5 + len(name)),
+                )
+
+    @support.cpython_only
+    def test_invalid_utf8_file_offset_after_non_ascii(self):
+        _testcapi = import_module('_testcapi')
+        self.addCleanup(unlink, TESTFN_ASCII)
+        with open(TESTFN_ASCII, 'wb') as f:
+            f.write(b'\nx = \xc3\xa9\xc3\xa9\xff\n')
         with self.assertRaises(SyntaxError) as caught:
-            compile(b"x = \xc3\xa9\xff\n", "<test>", "exec")
+            _testcapi.run_file(
+                os.fsencode(TESTFN_ASCII), _testcapi.Py_file_input, {})
         error = caught.exception
         self.assertEqual(
             (error.lineno, error.offset, error.end_lineno, error.end_offset),
-            (1, 6, 1, 6),
+            (2, 7, 2, 7),
         )
 
     def test_long_bom_conflict_message_is_not_truncated(self):

Makefile.pre.in

@@ -398,7 +398,7 @@ TOKENIZER_OBJS=		\
 		Parser/lexer/number.o \
 		Parser/lexer/state.o \
 		Parser/lexer/string.o \
-		Parser/tokenizer/cursor.o \
+		Parser/tokenizer/api.o \
 		Parser/tokenizer/decoder.o \
 		Parser/tokenizer/reader.o \
 		Parser/tokenizer/source.o \
@@ -410,10 +410,9 @@ PEGEN_HEADERS= \
 		$(srcdir)/Parser/string_parser.h
 
 TOKENIZER_HEADERS= \
-		Parser/lexer/lexer.h \
 		Parser/lexer/lexer_internal.h \
 		Parser/lexer/state.h \
-		Parser/tokenizer/cursor.h \
+		Parser/tokenizer/types.h \
 		Parser/tokenizer/reader.h \
 		Parser/tokenizer/reader_internal.h \
 		Parser/tokenizer/source.h \
@@ -3461,7 +3460,7 @@ MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo.
 MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_openssl_mem.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h
 MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/parts.h $(srcdir)/Modules/_testcapi/util.h
 MODULE__TESTLIMITEDCAPI_DEPS=$(srcdir)/Modules/_testlimitedcapi/testcapi_long.h $(srcdir)/Modules/_testlimitedcapi/parts.h $(srcdir)/Modules/_testlimitedcapi/util.h
-MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h $(srcdir)/Parser/tokenizer/cursor.h $(srcdir)/Parser/tokenizer/source.h $(srcdir)/Python/ceval.h $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h
+MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h $(srcdir)/Parser/tokenizer/source.h $(srcdir)/Parser/tokenizer/types.h $(srcdir)/Python/ceval.h $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h
 MODULE__SQLITE3_DEPS=$(srcdir)/Modules/_sqlite/connection.h $(srcdir)/Modules/_sqlite/cursor.h $(srcdir)/Modules/_sqlite/microprotocols.h $(srcdir)/Modules/_sqlite/module.h $(srcdir)/Modules/_sqlite/prepare_protocol.h $(srcdir)/Modules/_sqlite/row.h $(srcdir)/Modules/_sqlite/util.h
 MODULE__ZSTD_DEPS=$(srcdir)/Modules/_zstd/_zstdmodule.h $(srcdir)/Modules/_zstd/buffer.h $(srcdir)/Modules/_zstd/zstddict.h