diff --git a/Lib/test/test_capi/test_tokenizer.py b/Lib/test/test_capi/test_tokenizer.py
new file mode 100644
index 00000000000000..2fe1fef241e90a
--- /dev/null
+++ b/Lib/test/test_capi/test_tokenizer.py
@@ -0,0 +1,17 @@
+import unittest
+from test.support import import_helper
+
+
+_testinternalcapi = import_helper.import_module("_testinternalcapi")
+
+
+class TokenizerTests(unittest.TestCase):
+ def test_source(self):
+ _testinternalcapi.test_tokenizer_source()
+
+ def test_cursor(self):
+ _testinternalcapi.test_tokenizer_cursor()
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/Makefile.pre.in b/Makefile.pre.in
index b42a8491733c63..32f0c1326d3ead 100644
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -398,8 +398,10 @@ TOKENIZER_OBJS= \
Parser/lexer/number.o \
Parser/lexer/state.o \
Parser/lexer/string.o \
+ Parser/tokenizer/cursor.o \
Parser/tokenizer/file_tokenizer.o \
Parser/tokenizer/readline_tokenizer.o \
+ Parser/tokenizer/source.o \
Parser/tokenizer/string_tokenizer.o \
Parser/tokenizer/utf8_tokenizer.o \
Parser/tokenizer/helpers.o
@@ -414,6 +416,8 @@ TOKENIZER_HEADERS= \
Parser/lexer/lexer.h \
Parser/lexer/lexer_internal.h \
Parser/lexer/state.h \
+ Parser/tokenizer/cursor.h \
+ Parser/tokenizer/source.h \
Parser/tokenizer/tokenizer.h \
Parser/tokenizer/helpers.h
@@ -3479,7 +3483,7 @@ MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo.
MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h
MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/parts.h $(srcdir)/Modules/_testcapi/util.h
MODULE__TESTLIMITEDCAPI_DEPS=$(srcdir)/Modules/_testlimitedcapi/testcapi_long.h $(srcdir)/Modules/_testlimitedcapi/parts.h $(srcdir)/Modules/_testlimitedcapi/util.h
-MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h $(srcdir)/Python/ceval.h $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h
+MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h $(srcdir)/Parser/tokenizer/cursor.h $(srcdir)/Parser/tokenizer/source.h $(srcdir)/Python/ceval.h $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h
MODULE__SQLITE3_DEPS=$(srcdir)/Modules/_sqlite/connection.h $(srcdir)/Modules/_sqlite/cursor.h $(srcdir)/Modules/_sqlite/microprotocols.h $(srcdir)/Modules/_sqlite/module.h $(srcdir)/Modules/_sqlite/prepare_protocol.h $(srcdir)/Modules/_sqlite/row.h $(srcdir)/Modules/_sqlite/util.h
MODULE__ZSTD_DEPS=$(srcdir)/Modules/_zstd/_zstdmodule.h $(srcdir)/Modules/_zstd/buffer.h $(srcdir)/Modules/_zstd/zstddict.h
diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in
index 524f1466b191f6..0e6e8c938ec609 100644
--- a/Modules/Setup.stdlib.in
+++ b/Modules/Setup.stdlib.in
@@ -172,7 +172,7 @@
@MODULE_XXSUBTYPE_TRUE@xxsubtype xxsubtype.c
@MODULE__XXTESTFUZZ_TRUE@_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c
@MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c
-@MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/set.c _testinternalcapi/test_critical_sections.c _testinternalcapi/complex.c _testinternalcapi/interpreter.c _testinternalcapi/tuple.c
+@MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/set.c _testinternalcapi/test_critical_sections.c _testinternalcapi/complex.c _testinternalcapi/interpreter.c _testinternalcapi/tokenizer.c _testinternalcapi/tuple.c
@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/run.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c _testcapi/bytes.c _testcapi/object.c _testcapi/modsupport.c _testcapi/monitoring.c _testcapi/config.c _testcapi/import.c _testcapi/frame.c _testcapi/type.c _testcapi/function.c _testcapi/module.c _testcapi/weakref.c
@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/codec.c _testlimitedcapi/complex.c _testlimitedcapi/dict.c _testlimitedcapi/eval.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/import.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/object.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/slots.c _testlimitedcapi/sys.c _testlimitedcapi/threadstate.c _testlimitedcapi/tuple.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c _testlimitedcapi/version.c _testlimitedcapi/file.c _testlimitedcapi/weakref.c _testlimitedcapi/run.c
@MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c
diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c
index ea3ad2b81c2866..42e28318f4c0f4 100644
--- a/Modules/_testinternalcapi.c
+++ b/Modules/_testinternalcapi.c
@@ -3420,6 +3420,9 @@ module_exec(PyObject *module)
if (_PyTestInternalCapi_Init_CriticalSection(module) < 0) {
return 1;
}
+ if (_PyTestInternalCapi_Init_Tokenizer(module) < 0) {
+ return 1;
+ }
if (_PyTestInternalCapi_Init_Tuple(module) < 0) {
return 1;
}
diff --git a/Modules/_testinternalcapi/parts.h b/Modules/_testinternalcapi/parts.h
index 81f536c3babb18..67cf5e522766ca 100644
--- a/Modules/_testinternalcapi/parts.h
+++ b/Modules/_testinternalcapi/parts.h
@@ -15,6 +15,7 @@ int _PyTestInternalCapi_Init_PyTime(PyObject *module);
int _PyTestInternalCapi_Init_Set(PyObject *module);
int _PyTestInternalCapi_Init_Complex(PyObject *module);
int _PyTestInternalCapi_Init_CriticalSection(PyObject *module);
+int _PyTestInternalCapi_Init_Tokenizer(PyObject *module);
int _PyTestInternalCapi_Init_Tuple(PyObject *module);
#endif // Py_TESTINTERNALCAPI_PARTS_H
diff --git a/Modules/_testinternalcapi/tokenizer.c b/Modules/_testinternalcapi/tokenizer.c
new file mode 100644
index 00000000000000..72b7256adf42ee
--- /dev/null
+++ b/Modules/_testinternalcapi/tokenizer.c
@@ -0,0 +1,309 @@
+#include "parts.h"
+
+#include "../../Parser/tokenizer/cursor.h"
+
+static int
+check(int condition, const char *message)
+{
+ if (condition) {
+ return 0;
+ }
+ PyErr_SetString(PyExc_AssertionError, message);
+ return -1;
+}
+
+static int
+check_system_error(int failed, const char *message)
+{
+ if (!failed || !PyErr_ExceptionMatches(PyExc_SystemError)) {
+ PyErr_SetString(PyExc_AssertionError, message);
+ return -1;
+ }
+ PyErr_Clear();
+ return 0;
+}
+
+static int
+same_cursor(const _PyTok_Cursor *left, const _PyTok_Cursor *right)
+{
+ return left->source == right->source &&
+ left->pos == right->pos &&
+ left->line_start == right->line_start &&
+ left->line_end == right->line_end &&
+ left->lineno == right->lineno;
+}
+
+static PyObject *
+test_tokenizer_source(PyObject *Py_UNUSED(module),
+ PyObject *Py_UNUSED(args))
+{
+ _PyTok_SourceText source;
+ _PyTok_SourceInit(&source);
+
+ _PyTok_Loc loc;
+ _PyTok_Line line;
+ if (check(_PyTok_SourceLocation(
+ &source, 0, _PYTOK_AFFINITY_RIGHT, &loc) == 0,
+ "cannot locate empty source") < 0 ||
+ check(loc.lineno == 1 && loc.byte_col == 0,
+ "wrong empty source location") < 0 ||
+ check(_PyTok_SourceLine(&source, 1, &line) == 0,
+ "cannot find empty source line") < 0 ||
+ check(line.start == 0 && line.end == 0,
+ "wrong empty source line") < 0 ||
+ check_system_error(
+ _PyTok_SourceAppendLine(&source, "", 0, 0) < 0,
+ "accepted empty source line") < 0 ||
+ check_system_error(
+ _PyTok_SourceAppendLine(&source, "a\nb\n", 4, 0) < 0,
+ "accepted multiple source lines") < 0 ||
+ check_system_error(
+ _PyTok_SourceAppendLine(&source, "a", 1, 1) < 0,
+ "accepted missing implicit newline") < 0) {
+ goto error;
+ }
+
+ if (check(_PyTok_SourceAppendLine(&source, "alpha\n", 6, 0) == 0,
+ "wrong first source offset") < 0 ||
+ check(_PyTok_SourceAppendLine(
+ &source, "\xce\xb2\n", 3, 1) == 6,
+ "wrong second source offset") < 0 ||
+ check(_PyTok_SourceAppendLine(
+ &source, "nul\0x\n", 6, 0) == 9,
+ "wrong third source offset") < 0) {
+ goto error;
+ }
+
+ int marker_line = 257;
+ int final_line = 300;
+ _PyTok_Off marker_start = -1;
+ for (int lineno = 4; lineno <= final_line; lineno++) {
+ const char *text = lineno == marker_line ? "marker\n" : "x\n";
+ Py_ssize_t len = (Py_ssize_t)strlen(text);
+ _PyTok_Off start = _PyTok_SourceAppendLine(
+ &source, text, len, lineno == final_line);
+ if (start < 0) {
+ goto error;
+ }
+ if (lineno == marker_line) {
+ marker_start = start;
+ }
+ }
+
+ if (check(source.nlines == final_line, "wrong source line count") < 0 ||
+ check(_PyTok_SourceLine(&source, marker_line, &line) == 0,
+ "cannot find late source line") < 0 ||
+ check(line.start == marker_start &&
+ line.end == marker_start + 7,
+ "wrong late source line") < 0 ||
+ check(!line.implicit_newline && !line.contains_nul,
+ "wrong late source flags") < 0 ||
+ check(_PyTok_SourceLine(&source, 2, &line) == 0,
+ "cannot find second source line") < 0 ||
+ check(line.start == 6 && line.end == 9 &&
+ line.implicit_newline && !line.contains_nul,
+ "wrong second source line") < 0 ||
+ check(_PyTok_SourceLine(&source, 3, &line) == 0,
+ "cannot find third source line") < 0 ||
+ check(line.contains_nul, "missing null byte flag") < 0 ||
+ check(_PyTok_SourceLine(&source, final_line, &line) == 0,
+ "cannot find final source line") < 0 ||
+ check(line.implicit_newline,
+ "missing late implicit newline flag") < 0) {
+ goto error;
+ }
+
+ Py_ssize_t view_len;
+ const char *view = _PyTok_SourceSpanView(
+ &source, _PyTok_SpanFromBounds(6, 8), &view_len);
+ if (check(view != NULL && view_len == 2 &&
+ memcmp(view, "\xce\xb2", 2) == 0,
+ "wrong source span view") < 0 ||
+ check(_PyTok_SourceLocation(
+ &source, marker_start,
+ _PYTOK_AFFINITY_LEFT, &loc) == 0,
+ "cannot locate left line boundary") < 0 ||
+ check(loc.lineno == marker_line - 1 && loc.byte_col == 2,
+ "wrong left boundary location") < 0 ||
+ check(_PyTok_SourceLocation(
+ &source, marker_start,
+ _PYTOK_AFFINITY_RIGHT, &loc) == 0,
+ "cannot locate right line boundary") < 0 ||
+ check(loc.lineno == marker_line && loc.byte_col == 0,
+ "wrong right boundary location") < 0 ||
+ check(_PyTok_SourceLocation(
+ &source, marker_start + 1,
+ _PYTOK_AFFINITY_RIGHT, &loc) == 0,
+ "cannot locate late source byte") < 0 ||
+ check(loc.lineno == marker_line && loc.byte_col == 1,
+ "wrong late source location") < 0) {
+ goto error;
+ }
+
+ if (check(_PyTok_SourceLocation(
+ &source, source.len, _PYTOK_AFFINITY_LEFT, &loc) == 0,
+ "cannot locate left EOF") < 0 ||
+ check(loc.lineno == final_line && loc.byte_col == 2,
+ "wrong left EOF location") < 0 ||
+ check(_PyTok_SourceLocation(
+ &source, source.len,
+ _PYTOK_AFFINITY_RIGHT, &loc) == 0,
+ "cannot locate right EOF") < 0 ||
+ check(loc.lineno == final_line + 1 && loc.byte_col == 0,
+ "wrong right EOF location") < 0 ||
+ check(_PyTok_SourceLine(&source, final_line + 1, &line) == 0,
+ "cannot find virtual EOF line") < 0 ||
+ check(line.start == source.len && line.end == source.len,
+ "wrong virtual EOF line") < 0) {
+ goto error;
+ }
+
+ view = _PyTok_SourceSpanView(
+ &source, _PyTok_SpanFromBounds(0, source.len + 1), &view_len);
+ if (check_system_error(view == NULL, "accepted invalid source span") < 0 ||
+ check_system_error(
+ _PyTok_SourceLocation(
+ &source, source.len + 1,
+ _PYTOK_AFFINITY_RIGHT, &loc) < 0,
+ "accepted invalid source offset") < 0 ||
+ check_system_error(
+ _PyTok_SourceLine(&source, final_line + 2, &line) < 0,
+ "accepted invalid source line") < 0) {
+ goto error;
+ }
+
+ _PyTok_SourceClear(&source);
+ _PyTok_SourceInit(&source);
+ if (_PyTok_SourceAppendLine(&source, "tail", 4, 0) < 0 ||
+ check_system_error(
+ _PyTok_SourceAppendLine(&source, "x\n", 2, 0) < 0,
+ "appended after unterminated source line") < 0 ||
+ check(_PyTok_SourceLocation(
+ &source, source.len,
+ _PYTOK_AFFINITY_RIGHT, &loc) == 0,
+ "cannot locate unterminated EOF") < 0 ||
+ check(loc.lineno == 1 && loc.byte_col == 4,
+ "wrong unterminated EOF location") < 0) {
+ goto error;
+ }
+
+ _PyTok_SourceClear(&source);
+ Py_RETURN_NONE;
+
+error:
+ _PyTok_SourceClear(&source);
+ return NULL;
+}
+
+static PyObject *
+test_tokenizer_cursor(PyObject *Py_UNUSED(module),
+ PyObject *Py_UNUSED(args))
+{
+ _PyTok_SourceText source;
+ _PyTok_SourceInit(&source);
+ if (_PyTok_SourceAppendLine(&source, "ab\n", 3, 0) < 0 ||
+ _PyTok_SourceAppendLine(&source, "cd\n", 3, 0) < 0) {
+ goto error;
+ }
+
+ _PyTok_Cursor cursor;
+ _PyTok_CursorInit(&cursor, &source);
+ if (_PyTok_CursorSetOffset(&cursor, source.len) < 0 ||
+ check(cursor.lineno == 3 && cursor.pos == source.len,
+ "wrong cursor at virtual EOF") < 0 ||
+ _PyTok_CursorSetLine(&cursor, 1) < 0) {
+ goto error;
+ }
+
+ char large[BUFSIZ + 1];
+ memset(large, 'z', sizeof(large));
+ large[sizeof(large) - 1] = '\n';
+ if (_PyTok_SourceAppendLine(&source, large, sizeof(large), 0) < 0) {
+ goto error;
+ }
+
+ if (check(_PyTok_CursorPeek(&cursor, 0) == 'a',
+ "wrong cursor peek after relocation") < 0 ||
+ check(_PyTok_CursorPeek(&cursor, 1) == 'b',
+ "wrong distant cursor peek") < 0 ||
+ check(_PyTok_CursorAdvance(&cursor) == 'a',
+ "wrong first cursor byte") < 0 ||
+ check(_PyTok_CursorAdvance(&cursor) == 'b',
+ "wrong second cursor byte") < 0 ||
+ check(_PyTok_CursorAdvance(&cursor) == '\n',
+ "wrong final cursor byte") < 0 ||
+ check(_PyTok_CursorAdvance(&cursor) == EOF,
+ "cursor advanced past line") < 0 ||
+ check(_PyTok_CursorSetOffset(&cursor, 2) == 0,
+ "cannot seek cursor offset") < 0 ||
+ check(_PyTok_CursorAdvance(&cursor) == '\n',
+ "wrong cursor byte after seek") < 0 ||
+ check(_PyTok_CursorSetOffset(&cursor, 3) == 0,
+ "cannot seek line boundary") < 0 ||
+ check(cursor.lineno == 2 && cursor.line_start == 3 &&
+ _PyTok_CursorAdvance(&cursor) == 'c',
+ "wrong cursor at line boundary") < 0 ||
+ check(_PyTok_CursorSetLine(&cursor, 3) == 0,
+ "cannot advance cursor to final line") < 0 ||
+ check(cursor.line_start == 6 &&
+ _PyTok_CursorAdvance(&cursor) == 'z',
+ "wrong cursor byte on final line") < 0) {
+ goto error;
+ }
+
+ _PyTok_Cursor saved = cursor;
+ if (check_system_error(
+ _PyTok_CursorSetOffset(&cursor, source.len + 1) < 0,
+ "accepted invalid cursor offset") < 0 ||
+ check(same_cursor(&cursor, &saved),
+ "invalid offset changed cursor") < 0 ||
+ check_system_error(
+ _PyTok_CursorSetLine(&cursor, source.nlines + 2) < 0,
+ "accepted invalid cursor line") < 0 ||
+ check(same_cursor(&cursor, &saved),
+ "invalid line changed cursor") < 0 ||
+ check(_PyTok_CursorSetOffset(&cursor, source.len) == 0,
+ "cannot set cursor to EOF") < 0 ||
+ check(cursor.lineno == 4 && cursor.pos == source.len,
+ "wrong cursor at EOF") < 0) {
+ goto error;
+ }
+
+#if SIZEOF_VOID_P > 4
+ char byte = 0;
+ _PyTok_SourceText huge_source = {
+ .bytes = &byte,
+ .len = (_PyTok_Off)INT_MAX + 1,
+ };
+ _PyTok_Cursor huge_cursor = {
+ .source = &huge_source,
+ .pos = INT_MAX,
+ .line_end = (_PyTok_Off)INT_MAX + 1,
+ .lineno = 1,
+ };
+ if (check(_PyTok_CursorAdvance(&huge_cursor) == EOF &&
+ huge_cursor.pos == INT_MAX,
+ "cursor advanced past maximum column") < 0) {
+ goto error;
+ }
+#endif
+
+ _PyTok_SourceClear(&source);
+ Py_RETURN_NONE;
+
+error:
+ _PyTok_SourceClear(&source);
+ return NULL;
+}
+
+static PyMethodDef test_methods[] = {
+ {"test_tokenizer_source", test_tokenizer_source, METH_NOARGS},
+ {"test_tokenizer_cursor", test_tokenizer_cursor, METH_NOARGS},
+ {NULL},
+};
+
+int
+_PyTestInternalCapi_Init_Tokenizer(PyObject *module)
+{
+ return PyModule_AddFunctions(module, test_methods);
+}
diff --git a/PCbuild/_testinternalcapi.vcxproj b/PCbuild/_testinternalcapi.vcxproj
index f3e423fa04668e..f89ed488dc0902 100644
--- a/PCbuild/_testinternalcapi.vcxproj
+++ b/PCbuild/_testinternalcapi.vcxproj
@@ -100,6 +100,7 @@
+
diff --git a/PCbuild/_testinternalcapi.vcxproj.filters b/PCbuild/_testinternalcapi.vcxproj.filters
index 7ab242c2c230b6..f1b34ad886f1f9 100644
--- a/PCbuild/_testinternalcapi.vcxproj.filters
+++ b/PCbuild/_testinternalcapi.vcxproj.filters
@@ -27,6 +27,9 @@
Source Files
+
+ Source Files
+
Source Files
diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj
index 47296a93424a8f..7a311dd5388c15 100644
--- a/PCbuild/pythoncore.vcxproj
+++ b/PCbuild/pythoncore.vcxproj
@@ -422,6 +422,8 @@
+
+
@@ -588,6 +590,8 @@
+
+
diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters
index 124f6c7bb90de6..403e571a2ff8d1 100644
--- a/PCbuild/pythoncore.vcxproj.filters
+++ b/PCbuild/pythoncore.vcxproj.filters
@@ -333,6 +333,12 @@
Parser
+
+ Parser
+
+
+ Parser
+
Parser
@@ -1346,6 +1352,12 @@
Parser
+
+ Parser
+
+
+ Parser
+
Parser
diff --git a/Parser/tokenizer/cursor.c b/Parser/tokenizer/cursor.c
new file mode 100644
index 00000000000000..66978a7b618574
--- /dev/null
+++ b/Parser/tokenizer/cursor.c
@@ -0,0 +1,89 @@
+#include "Python.h"
+
+#include "cursor.h"
+
+static void
+set_line(_PyTok_Cursor *cursor, int lineno, _PyTok_Off start,
+ _PyTok_Off end)
+{
+ cursor->pos = start;
+ cursor->line_start = start;
+ cursor->line_end = end;
+ cursor->lineno = lineno;
+}
+
+int
+_PyTok_CursorSetLine(_PyTok_Cursor *cursor, int lineno)
+{
+ if (cursor->source == NULL) {
+ PyErr_SetString(PyExc_SystemError, "cursor has no tokenizer source");
+ return -1;
+ }
+ const _PyTok_SourceText *source = cursor->source;
+ if (lineno > 0 && cursor->lineno == lineno - 1 &&
+ lineno <= source->nlines) {
+ _PyTok_Off start = cursor->line_end;
+ _PyTok_Off end = source->len;
+ if (lineno < source->nlines) {
+ const char *newline = memchr(
+ source->bytes + start, '\n', source->len - start);
+ if (newline == NULL) {
+ PyErr_SetString(PyExc_SystemError,
+ "corrupt tokenizer source line index");
+ return -1;
+ }
+ end = newline - source->bytes + 1;
+ }
+ set_line(cursor, lineno, start, end);
+ return 0;
+ }
+
+ _PyTok_Line line;
+ if (_PyTok_SourceLine(source, lineno, &line) < 0) {
+ return -1;
+ }
+ set_line(cursor, lineno, line.start, line.end);
+ return 0;
+}
+
+int
+_PyTok_CursorSetOffset(_PyTok_Cursor *cursor, _PyTok_Off offset)
+{
+ if (cursor->source == NULL) {
+ PyErr_SetString(PyExc_SystemError, "cursor has no tokenizer source");
+ return -1;
+ }
+ const _PyTok_SourceText *source = cursor->source;
+ int stays_on_line = cursor->lineno > 0 &&
+ offset >= cursor->line_start && offset < cursor->line_end;
+ if (!stays_on_line && cursor->lineno > 0 &&
+ offset == cursor->line_end && offset == source->len &&
+ (offset == 0 || source->bytes[offset - 1] != '\n')) {
+ stays_on_line = 1;
+ }
+ if (stays_on_line) {
+ cursor->pos = offset;
+ return 0;
+ }
+
+ _PyTok_Loc loc;
+ if (_PyTok_SourceLocation(
+ source, offset, _PYTOK_AFFINITY_RIGHT, &loc) < 0) {
+ return -1;
+ }
+ _PyTok_Off start = offset - loc.byte_col;
+ _PyTok_Off end = source->len;
+ if (loc.lineno < source->nlines) {
+ const char *newline = memchr(
+ source->bytes + start, '\n', source->len - start);
+ if (newline == NULL) {
+ PyErr_SetString(PyExc_SystemError,
+ "corrupt tokenizer source line index");
+ return -1;
+ }
+ end = newline - source->bytes + 1;
+ }
+ set_line(cursor, loc.lineno, start, end);
+ cursor->pos = offset;
+ return 0;
+}
diff --git a/Parser/tokenizer/cursor.h b/Parser/tokenizer/cursor.h
new file mode 100644
index 00000000000000..710f59fcd8598c
--- /dev/null
+++ b/Parser/tokenizer/cursor.h
@@ -0,0 +1,58 @@
+#ifndef Py_TOKENIZER_CURSOR_H
+#define Py_TOKENIZER_CURSOR_H
+
+#include "source.h"
+
+typedef struct {
+ /* The source must remain initialized at this address while in use. */
+ const _PyTok_SourceText *source;
+ _PyTok_Off pos;
+ _PyTok_Off line_start;
+ _PyTok_Off line_end;
+ int lineno;
+} _PyTok_Cursor;
+
+PyAPI_FUNC(int) _PyTok_CursorSetLine(_PyTok_Cursor *, int);
+/* A line-boundary offset selects the following line. */
+PyAPI_FUNC(int) _PyTok_CursorSetOffset(_PyTok_Cursor *, _PyTok_Off);
+
+static inline void
+_PyTok_CursorInit(_PyTok_Cursor *cursor, const _PyTok_SourceText *source)
+{
+ *cursor = (_PyTok_Cursor){
+ .source = source,
+ };
+}
+
+/* EOF with pos before line_end reports byte-column overflow. */
+static inline int
+_PyTok_CursorAdvance(_PyTok_Cursor *cursor)
+{
+ assert(cursor->source != NULL);
+ assert(cursor->pos >= cursor->line_start);
+ assert(cursor->pos <= cursor->line_end);
+ assert(cursor->line_end <= cursor->source->len);
+ if (cursor->pos >= cursor->line_end) {
+ return EOF;
+ }
+ if (cursor->pos - cursor->line_start >= INT_MAX) {
+ return EOF;
+ }
+ return Py_CHARMASK(cursor->source->bytes[cursor->pos++]);
+}
+
+static inline int
+_PyTok_CursorPeek(const _PyTok_Cursor *cursor, int distance)
+{
+ assert(cursor->source != NULL);
+ assert(cursor->pos >= cursor->line_start);
+ assert(cursor->pos <= cursor->line_end);
+ assert(cursor->line_end <= cursor->source->len);
+ if ((size_t)distance >=
+ (size_t)(cursor->line_end - cursor->pos)) {
+ return EOF;
+ }
+ return Py_CHARMASK(cursor->source->bytes[cursor->pos + distance]);
+}
+
+#endif
diff --git a/Parser/tokenizer/source.c b/Parser/tokenizer/source.c
new file mode 100644
index 00000000000000..e754c71b2e45f8
--- /dev/null
+++ b/Parser/tokenizer/source.c
@@ -0,0 +1,331 @@
+#include "Python.h"
+
+#include "source.h"
+
+#define LINE_CHECKPOINT_INTERVAL 256
+
+void
+_PyTok_SourceInit(_PyTok_SourceText *source)
+{
+ *source = (_PyTok_SourceText){0};
+}
+
+void
+_PyTok_SourceClear(_PyTok_SourceText *source)
+{
+ PyMem_Free(source->bytes);
+ PyMem_Free(source->line_checkpoints);
+ PyMem_Free(source->implicit_lines);
+ _PyTok_SourceInit(source);
+}
+
+static int
+reserve_bytes(_PyTok_SourceText *source, Py_ssize_t needed)
+{
+ if (needed <= source->cap) {
+ return 0;
+ }
+ Py_ssize_t cap = source->cap > 0 ? source->cap : BUFSIZ;
+ while (cap < needed) {
+ if (cap > PY_SSIZE_T_MAX / 2) {
+ cap = needed;
+ break;
+ }
+ cap *= 2;
+ }
+ char *bytes;
+#ifdef Py_DEBUG
+ /* Moving on growth makes stale interior pointers fail in debug builds. */
+ bytes = PyMem_Malloc(cap);
+ if (bytes != NULL && source->len > 0) {
+ memcpy(bytes, source->bytes, source->len);
+ }
+#else
+ bytes = PyMem_Realloc(source->bytes, cap);
+#endif
+ if (bytes == NULL) {
+ PyErr_NoMemory();
+ return -1;
+ }
+#ifdef Py_DEBUG
+ if (source->bytes != NULL) {
+ memset(source->bytes, 0xDD, source->cap);
+ PyMem_Free(source->bytes);
+ }
+#endif
+ source->bytes = bytes;
+ source->cap = cap;
+ return 0;
+}
+
+static int
+reserve_checkpoints(_PyTok_SourceText *source, int needed)
+{
+ if (needed <= source->checkpoints_cap) {
+ return 0;
+ }
+ int cap;
+ if (source->checkpoints_cap == 0) {
+ cap = 16;
+ }
+ else if (source->checkpoints_cap <= INT_MAX / 2) {
+ cap = source->checkpoints_cap * 2;
+ }
+ else {
+ PyErr_NoMemory();
+ return -1;
+ }
+ if ((size_t)cap > (size_t)PY_SSIZE_T_MAX /
+ sizeof(*source->line_checkpoints)) {
+ PyErr_NoMemory();
+ return -1;
+ }
+ _PyTok_Off *checkpoints = PyMem_Realloc(
+ source->line_checkpoints,
+ (size_t)cap * sizeof(*source->line_checkpoints));
+ if (checkpoints == NULL) {
+ PyErr_NoMemory();
+ return -1;
+ }
+ source->line_checkpoints = checkpoints;
+ source->checkpoints_cap = cap;
+ return 0;
+}
+
+static int
+reserve_implicit_lines(_PyTok_SourceText *source, int nlines)
+{
+ Py_ssize_t needed = ((Py_ssize_t)nlines + 7) / 8;
+ if (needed <= source->implicit_cap) {
+ return 0;
+ }
+ Py_ssize_t cap = source->implicit_cap > 0 ? source->implicit_cap : 16;
+ while (cap < needed) {
+ if (cap > PY_SSIZE_T_MAX / 2) {
+ cap = needed;
+ break;
+ }
+ cap *= 2;
+ }
+ unsigned char *lines = PyMem_Realloc(source->implicit_lines, cap);
+ if (lines == NULL) {
+ PyErr_NoMemory();
+ return -1;
+ }
+ memset(lines + source->implicit_cap, 0, cap - source->implicit_cap);
+ source->implicit_lines = lines;
+ source->implicit_cap = cap;
+ return 0;
+}
+
+static int
+validate_line(const _PyTok_SourceText *source, const char *bytes,
+ Py_ssize_t len, int implicit_newline)
+{
+ if (len <= 0 || bytes == NULL ||
+ (source->nlines > 0 &&
+ (source->len == 0 || source->bytes[source->len - 1] != '\n'))) {
+ PyErr_SetString(PyExc_SystemError, "invalid tokenizer source line");
+ return -1;
+ }
+ const char *newline = memchr(bytes, '\n', len);
+ if ((newline != NULL && newline != bytes + len - 1) ||
+ (implicit_newline && newline == NULL)) {
+ PyErr_SetString(PyExc_SystemError, "invalid tokenizer source line");
+ return -1;
+ }
+ if (source->nlines == INT_MAX ||
+ (source->nlines == INT_MAX - 1 && newline != NULL)) {
+ PyErr_SetString(PyExc_OverflowError, "too many tokenizer source lines");
+ return -1;
+ }
+ return 0;
+}
+
+_PyTok_Off
+_PyTok_SourceAppendLine(_PyTok_SourceText *source, const char *bytes,
+ Py_ssize_t len, int implicit_newline)
+{
+ if (validate_line(source, bytes, len, implicit_newline) < 0) {
+ return -1;
+ }
+ if (source->len > PY_SSIZE_T_MAX - len - 1) {
+ PyErr_NoMemory();
+ return -1;
+ }
+ int nlines = source->nlines + 1;
+ int checkpoint = ((nlines - 1) % LINE_CHECKPOINT_INTERVAL) == 0;
+ int checkpoint_count = (nlines - 1) / LINE_CHECKPOINT_INTERVAL + 1;
+ if ((checkpoint &&
+ reserve_checkpoints(source, checkpoint_count) < 0) ||
+ (implicit_newline && reserve_implicit_lines(source, nlines) < 0) ||
+ reserve_bytes(source, source->len + len + 1) < 0) {
+ return -1;
+ }
+
+ _PyTok_Off start = source->len;
+ memcpy(source->bytes + start, bytes, len);
+ source->len += len;
+ source->bytes[source->len] = '\0';
+ if (checkpoint) {
+ source->line_checkpoints[checkpoint_count - 1] = start;
+ }
+ if (implicit_newline) {
+ source->implicit_lines[(nlines - 1) / 8] |=
+ (unsigned char)(1U << ((nlines - 1) & 7));
+ }
+ source->nlines = nlines;
+ return start;
+}
+
+const char *
+_PyTok_SourceSpanView(const _PyTok_SourceText *source, _PyTok_Span span,
+ Py_ssize_t *len)
+{
+ if (!_PyTok_SpanIsValid(span) || span.end > source->len || len == NULL) {
+ PyErr_SetString(PyExc_SystemError, "invalid tokenizer source span");
+ return NULL;
+ }
+ *len = span.end - span.start;
+ return source->bytes == NULL ? "" : source->bytes + span.start;
+}
+
+static int
+line_is_implicit(const _PyTok_SourceText *source, int lineno)
+{
+ if ((lineno - 1) / 8 >= source->implicit_cap) {
+ return 0;
+ }
+ return (source->implicit_lines[(lineno - 1) / 8] >>
+ ((lineno - 1) & 7)) & 1;
+}
+
+static int
+source_ends_in_newline(const _PyTok_SourceText *source)
+{
+ return source->len > 0 && source->bytes[source->len - 1] == '\n';
+}
+
+static int
+eof_lineno(const _PyTok_SourceText *source)
+{
+ if (source->nlines == 0) {
+ return 1;
+ }
+ return source->nlines + source_ends_in_newline(source);
+}
+
+int
+_PyTok_SourceLine(const _PyTok_SourceText *source, int lineno,
+ _PyTok_Line *line)
+{
+ if (line == NULL || lineno < 1 || lineno > eof_lineno(source)) {
+ PyErr_SetString(PyExc_SystemError, "invalid tokenizer source line");
+ return -1;
+ }
+ if (lineno > source->nlines) {
+ *line = (_PyTok_Line){
+ .start = source->len,
+ .end = source->len,
+ };
+ return 0;
+ }
+
+ int checkpoint = (lineno - 1) / LINE_CHECKPOINT_INTERVAL;
+ int current = checkpoint * LINE_CHECKPOINT_INTERVAL + 1;
+ _PyTok_Off start = source->line_checkpoints[checkpoint];
+ while (current < lineno) {
+ const char *newline = memchr(
+ source->bytes + start, '\n', source->len - start);
+ if (newline == NULL) {
+ PyErr_SetString(PyExc_SystemError,
+ "corrupt tokenizer source line index");
+ return -1;
+ }
+ start = newline - source->bytes + 1;
+ current++;
+ }
+ _PyTok_Off end = source->len;
+ if (lineno < source->nlines) {
+ const char *newline = memchr(
+ source->bytes + start, '\n', source->len - start);
+ if (newline == NULL) {
+ PyErr_SetString(PyExc_SystemError,
+ "corrupt tokenizer source line index");
+ return -1;
+ }
+ end = newline - source->bytes + 1;
+ }
+ *line = (_PyTok_Line){
+ .start = start,
+ .end = end,
+ .implicit_newline = line_is_implicit(source, lineno),
+ .contains_nul = memchr(
+ source->bytes + start, 0, end - start) != NULL,
+ };
+ return 0;
+}
+
+int
+_PyTok_SourceLocation(const _PyTok_SourceText *source, _PyTok_Off offset,
+ _PyTok_Affinity affinity, _PyTok_Loc *loc)
+{
+ if (offset < 0 || offset > source->len || loc == NULL ||
+ (affinity != _PYTOK_AFFINITY_LEFT &&
+ affinity != _PYTOK_AFFINITY_RIGHT)) {
+ PyErr_SetString(PyExc_SystemError, "invalid tokenizer source offset");
+ return -1;
+ }
+ if (source->nlines == 0 ||
+ (offset == source->len && source_ends_in_newline(source) &&
+ affinity == _PYTOK_AFFINITY_RIGHT)) {
+ *loc = (_PyTok_Loc){eof_lineno(source), 0};
+ return 0;
+ }
+
+ _PyTok_Off key = offset;
+ if (affinity == _PYTOK_AFFINITY_LEFT && key > 0) {
+ key--;
+ }
+ int low = 0;
+ int high = (source->nlines - 1) / LINE_CHECKPOINT_INTERVAL + 1;
+ while (low < high) {
+ int middle = low + (high - low) / 2;
+ if (source->line_checkpoints[middle] <= key) {
+ low = middle + 1;
+ }
+ else {
+ high = middle;
+ }
+ }
+ int checkpoint = low - 1;
+ if (checkpoint < 0) {
+ PyErr_SetString(PyExc_SystemError, "corrupt tokenizer source line index");
+ return -1;
+ }
+ int lineno = checkpoint * LINE_CHECKPOINT_INTERVAL + 1;
+ _PyTok_Off start = source->line_checkpoints[checkpoint];
+ while (lineno < source->nlines) {
+ const char *newline = memchr(
+ source->bytes + start, '\n', source->len - start);
+ if (newline == NULL) {
+ PyErr_SetString(PyExc_SystemError,
+ "corrupt tokenizer source line index");
+ return -1;
+ }
+ _PyTok_Off end = newline - source->bytes + 1;
+ if (offset < end ||
+ (offset == end && affinity == _PYTOK_AFFINITY_LEFT)) {
+ break;
+ }
+ start = end;
+ lineno++;
+ }
+ _PyTok_Off byte_col = offset - start;
+ if (byte_col > INT_MAX) {
+ PyErr_SetString(PyExc_OverflowError, "tokenizer column is too large");
+ return -1;
+ }
+ *loc = (_PyTok_Loc){lineno, (int)byte_col};
+ return 0;
+}
diff --git a/Parser/tokenizer/source.h b/Parser/tokenizer/source.h
new file mode 100644
index 00000000000000..d52a5566e0e6ec
--- /dev/null
+++ b/Parser/tokenizer/source.h
@@ -0,0 +1,70 @@
+#ifndef Py_TOKENIZER_SOURCE_H
+#define Py_TOKENIZER_SOURCE_H
+
+#include "Python.h"
+
+typedef Py_ssize_t _PyTok_Off;
+
+typedef struct {
+ _PyTok_Off start;
+ _PyTok_Off end;
+} _PyTok_Span;
+
+typedef struct {
+ int lineno;
+ int byte_col;
+} _PyTok_Loc;
+
+typedef enum {
+ _PYTOK_AFFINITY_LEFT,
+ _PYTOK_AFFINITY_RIGHT,
+} _PyTok_Affinity;
+
+typedef struct {
+ _PyTok_Off start;
+ _PyTok_Off end;
+ unsigned implicit_newline : 1;
+ unsigned contains_nul : 1;
+} _PyTok_Line;
+
+typedef struct {
+ char *bytes;
+ _PyTok_Off len;
+ _PyTok_Off cap;
+ _PyTok_Off *line_checkpoints;
+ unsigned char *implicit_lines;
+ int nlines;
+ int checkpoints_cap;
+ Py_ssize_t implicit_cap;
+} _PyTok_SourceText;
+
+PyAPI_FUNC(void) _PyTok_SourceInit(_PyTok_SourceText *);
+/* Clear invalidates all cursors and span views for the source. */
+PyAPI_FUNC(void) _PyTok_SourceClear(_PyTok_SourceText *);
+/* Append one logical line. The implicit_newline flag means that its newline
+ terminator was synthesized. The input must not point into source storage. */
+PyAPI_FUNC(_PyTok_Off) _PyTok_SourceAppendLine(
+ _PyTok_SourceText *source, const char *bytes, Py_ssize_t len,
+ int implicit_newline);
+/* The returned view is invalidated by SourceAppendLine and SourceClear. */
+PyAPI_FUNC(const char *) _PyTok_SourceSpanView(
+ const _PyTok_SourceText *, _PyTok_Span, Py_ssize_t *);
+PyAPI_FUNC(int) _PyTok_SourceLine(
+ const _PyTok_SourceText *, int, _PyTok_Line *);
+/* At a line boundary, affinity selects the preceding or following line. */
+PyAPI_FUNC(int) _PyTok_SourceLocation(
+ const _PyTok_SourceText *, _PyTok_Off, _PyTok_Affinity, _PyTok_Loc *);
+
+static inline _PyTok_Span
+_PyTok_SpanFromBounds(_PyTok_Off start, _PyTok_Off end)
+{
+ return (_PyTok_Span){start, end};
+}
+
+static inline int
+_PyTok_SpanIsValid(_PyTok_Span span)
+{
+ return span.start >= 0 && span.end >= span.start;
+}
+
+#endif