Skip to content

[mypyc] Make list get/set item more memory safe on free-threaded builds #21683

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
JukkaL merged 2 commits into from
Jul 6, 2026
+213 −62
Merged

[mypyc] Make list get/set item more memory safe on free-threaded builds #21683

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ca88569
[mypyc] Make list get item memory safe on free-threaded builds
JukkaL Jul 4, 2026
d30ba7c
[mypyc] Make list set item memory safe on free-threaded builds
JukkaL Jul 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 56 additions & 4 deletions mypyc/lib-rt/CPy.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -664,16 +664,68 @@ PyObject *CPyObject_GetSlice(PyObject *obj, CPyTagged start, CPyTagged end);
// List operations


PyObject *CPyList_Build(Py_ssize_t len, ...);
#ifndef Py_GIL_DISABLED

PyObject *CPyList_GetItem(PyObject *list, CPyTagged index);
PyObject *CPyList_GetItemShort(PyObject *list, CPyTagged index);
PyObject *CPyList_GetItemInt64(PyObject *list, int64_t index);
bool CPyList_SetItem(PyObject *list, CPyTagged index, PyObject *value);
bool CPyList_SetItemInt64(PyObject *list, int64_t index, PyObject *value);

#else

PyObject *CPyList_GetItem_(PyObject *list, CPyTagged index);
bool CPyList_SetItem_(PyObject *list, CPyTagged index, PyObject *value);

static inline PyObject *CPyList_GetItem(PyObject *list, CPyTagged index) {
if (likely(CPyTagged_CheckShort(index) && !CPyTagged_IsNegative(index))) {
// Inlined fast path
Py_ssize_t n = CPyTagged_ShortAsSsize_t(index);
return PyList_GetItemRef(list, n);
} else {
return CPyList_GetItem_(list, index);
}
}

static inline PyObject *CPyList_GetItemShort(PyObject *list, CPyTagged index) {
Py_ssize_t n = CPyTagged_ShortAsSsize_t(index);
if (n < 0) {
n += PyList_GET_SIZE(list);
}
return PyList_GetItemRef(list, n);
}

static inline PyObject *CPyList_GetItemInt64(PyObject *list, int64_t index) {
if (index < 0) {
index += PyList_GET_SIZE(list);
}
return PyList_GetItemRef(list, index);
}

static inline bool CPyList_SetItem(PyObject *list, CPyTagged index, PyObject *value) {
if (likely(CPyTagged_CheckShort(index) && !CPyTagged_IsNegative(index))) {
// Inlined fast path
Py_ssize_t n = CPyTagged_ShortAsSsize_t(index);
return PyList_SetItem(list, n, value) >= 0;
} else {
return CPyList_SetItem_(list, index, value);
}
}

static inline bool CPyList_SetItemInt64(PyObject *list, int64_t index, PyObject *value) {
if (index < 0) {
index += PyList_GET_SIZE(list);
}
return PyList_SetItem(list, index, value) >= 0;
}

#endif

PyObject *CPyList_GetItemBorrow(PyObject *list, CPyTagged index);
PyObject *CPyList_GetItemShortBorrow(PyObject *list, CPyTagged index);
PyObject *CPyList_GetItemInt64(PyObject *list, int64_t index);
PyObject *CPyList_GetItemInt64Borrow(PyObject *list, int64_t index);
bool CPyList_SetItem(PyObject *list, CPyTagged index, PyObject *value);
void CPyList_SetItemUnsafe(PyObject *list, Py_ssize_t index, PyObject *value);
bool CPyList_SetItemInt64(PyObject *list, int64_t index, PyObject *value);
PyObject *CPyList_Build(Py_ssize_t len, ...);
PyObject *CPyList_PopLast(PyObject *obj);
PyObject *CPyList_Pop(PyObject *obj, CPyTagged index);
CPyTagged CPyList_Count(PyObject *obj, PyObject *value);
Expand Down
149 changes: 91 additions & 58 deletions mypyc/lib-rt/list_ops.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,8 @@ PyObject *CPyList_Copy(PyObject *list) {
return PyObject_CallMethodNoArgs(list, mypyc_interned_str.copy);
}

#ifndef Py_GIL_DISABLED

PyObject *CPyList_GetItemShort(PyObject *list, CPyTagged index) {
Py_ssize_t n = CPyTagged_ShortAsSsize_t(index);
Py_ssize_t size = PyList_GET_SIZE(list);
Expand All @@ -69,24 +71,6 @@ PyObject *CPyList_GetItemShort(PyObject *list, CPyTagged index) {
return result;
}

PyObject *CPyList_GetItemShortBorrow(PyObject *list, CPyTagged index) {
Py_ssize_t n = CPyTagged_ShortAsSsize_t(index);
Py_ssize_t size = PyList_GET_SIZE(list);
if (n >= 0) {
if (n >= size) {
PyErr_SetString(PyExc_IndexError, "list index out of range");
return NULL;
}
} else {
n += size;
if (n < 0) {
PyErr_SetString(PyExc_IndexError, "list index out of range");
return NULL;
}
}
return PyList_GET_ITEM(list, n);
}

PyObject *CPyList_GetItem(PyObject *list, CPyTagged index) {
if (CPyTagged_CheckShort(index)) {
Py_ssize_t n = CPyTagged_ShortAsSsize_t(index);
Expand All @@ -112,29 +96,6 @@ PyObject *CPyList_GetItem(PyObject *list, CPyTagged index) {
}
}

PyObject *CPyList_GetItemBorrow(PyObject *list, CPyTagged index) {
if (CPyTagged_CheckShort(index)) {
Py_ssize_t n = CPyTagged_ShortAsSsize_t(index);
Py_ssize_t size = PyList_GET_SIZE(list);
if (n >= 0) {
if (n >= size) {
PyErr_SetString(PyExc_IndexError, "list index out of range");
return NULL;
}
} else {
n += size;
if (n < 0) {
PyErr_SetString(PyExc_IndexError, "list index out of range");
return NULL;
}
}
return PyList_GET_ITEM(list, n);
} else {
PyErr_SetString(PyExc_OverflowError, CPYTHON_LARGE_INT_ERRMSG);
return NULL;
}
}

PyObject *CPyList_GetItemInt64(PyObject *list, int64_t index) {
size_t size = PyList_GET_SIZE(list);
if (likely((uint64_t)index < size)) {
Expand All @@ -156,23 +117,6 @@ PyObject *CPyList_GetItemInt64(PyObject *list, int64_t index) {
return result;
}

PyObject *CPyList_GetItemInt64Borrow(PyObject *list, int64_t index) {
size_t size = PyList_GET_SIZE(list);
if (likely((uint64_t)index < size)) {
return PyList_GET_ITEM(list, index);
}
if (index >= 0) {
PyErr_SetString(PyExc_IndexError, "list index out of range");
return NULL;
}
index += size;
if (index < 0) {
PyErr_SetString(PyExc_IndexError, "list index out of range");
return NULL;
}
return PyList_GET_ITEM(list, index);
}

bool CPyList_SetItem(PyObject *list, CPyTagged index, PyObject *value) {
if (CPyTagged_CheckShort(index)) {
Py_ssize_t n = CPyTagged_ShortAsSsize_t(index);
Expand Down Expand Up @@ -220,6 +164,95 @@ bool CPyList_SetItemInt64(PyObject *list, int64_t index, PyObject *value) {
return true;
}

#else /* Py_GIL_DISABLED */

PyObject *CPyList_GetItem_(PyObject *list, CPyTagged index) {
if (CPyTagged_CheckShort(index)) {
Py_ssize_t n = CPyTagged_ShortAsSsize_t(index);
if (n < 0) {
n += PyList_GET_SIZE(list);
}
return PyList_GetItemRef(list, n);
} else {
PyErr_SetString(PyExc_OverflowError, CPYTHON_LARGE_INT_ERRMSG);
return NULL;
}
}

bool CPyList_SetItem_(PyObject *list, CPyTagged index, PyObject *value) {
if (CPyTagged_CheckShort(index)) {
Py_ssize_t n = CPyTagged_ShortAsSsize_t(index);
Py_ssize_t size = PyList_GET_SIZE(list);
if (n < 0) {
n += size;
}
return PyList_SetItem(list, n, value) >= 0;
} else {
PyErr_SetString(PyExc_OverflowError, CPYTHON_LARGE_INT_ERRMSG);
return false;
}
}

#endif

PyObject *CPyList_GetItemShortBorrow(PyObject *list, CPyTagged index) {
Py_ssize_t n = CPyTagged_ShortAsSsize_t(index);
Py_ssize_t size = PyList_GET_SIZE(list);
if (n >= 0) {
if (n >= size) {
PyErr_SetString(PyExc_IndexError, "list index out of range");
return NULL;
}
} else {
n += size;
if (n < 0) {
PyErr_SetString(PyExc_IndexError, "list index out of range");
return NULL;
}
}
return PyList_GET_ITEM(list, n);
}

PyObject *CPyList_GetItemBorrow(PyObject *list, CPyTagged index) {
if (CPyTagged_CheckShort(index)) {
Py_ssize_t n = CPyTagged_ShortAsSsize_t(index);
Py_ssize_t size = PyList_GET_SIZE(list);
if (n >= 0) {
if (n >= size) {
PyErr_SetString(PyExc_IndexError, "list index out of range");
return NULL;
}
} else {
n += size;
if (n < 0) {
PyErr_SetString(PyExc_IndexError, "list index out of range");
return NULL;
}
}
return PyList_GET_ITEM(list, n);
} else {
PyErr_SetString(PyExc_OverflowError, CPYTHON_LARGE_INT_ERRMSG);
return NULL;
}
}

PyObject *CPyList_GetItemInt64Borrow(PyObject *list, int64_t index) {
size_t size = PyList_GET_SIZE(list);
if (likely((uint64_t)index < size)) {
return PyList_GET_ITEM(list, index);
}
if (index >= 0) {
PyErr_SetString(PyExc_IndexError, "list index out of range");
return NULL;
}
index += size;
if (index < 0) {
PyErr_SetString(PyExc_IndexError, "list index out of range");
return NULL;
}
return PyList_GET_ITEM(list, index);
}

// This function should only be used to fill in brand new lists.
void CPyList_SetItemUnsafe(PyObject *list, Py_ssize_t index, PyObject *value) {
PyList_SET_ITEM(list, index, value);
Expand Down
4 changes: 4 additions & 0 deletions mypyc/lib-rt/mypyc_util.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,10 @@ static inline CPyTagged CPyTagged_ShortFromSsize_t(Py_ssize_t x) {
return x << 1;
}

static inline int CPyTagged_IsNegative(CPyTagged x) {
return ((Py_ssize_t)x) < 0;
}

// Are we targeting Python 3.X or newer?
#define CPY_3_11_FEATURES (PY_VERSION_HEX >= 0x030b0000)
#define CPY_3_12_FEATURES (PY_VERSION_HEX >= 0x030c0000)
Expand Down
62 changes: 62 additions & 0 deletions mypyc/test-data/run-lists.test
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,68 @@ def test_list_build() -> None:
l3.append('a')
assert l3 == ['a']

class C:
def __init__(self, s: str) -> None:
self.s = s

a = [C("a"), C("b"), C("c")]

def test_get_item() -> None:
zero = int()
assert a[zero].s == "a"
assert a[zero + 1].s == "b"
assert a[zero + 2].s == "c"
assert a[zero - 1].s == "c"
assert a[zero - 2].s == "b"
assert a[zero - 3].s == "a"
with assertRaises(IndexError):
a[zero + 3]
with assertRaises(IndexError):
a[zero - 4]
# TODO: Raise IndexError?
with assertRaises(OverflowError):
a[zero + 2**90]
with assertRaises(OverflowError):
a[zero - 2**90]

def test_get_item_literal() -> None:
assert a[0].s == "a"
assert a[1].s == "b"
assert a[2].s == "c"
assert a[-1].s == "c"
assert a[-2].s == "b"
assert a[-3].s == "a"
with assertRaises(IndexError):
a[3]
with assertRaises(IndexError):
a[-4]
# TODO: Raise IndexError?
with assertRaises(OverflowError):
a[2**90]
with assertRaises(OverflowError):
a[-2**90]

def test_set_item() -> None:
l = a.copy()
zero = int()
zero2 = int()
l[zero] = C("x")
assert l[zero2].s == "x"
l[zero + 2] = C("y")
assert l[zero2 + 2].s == "y"
l[zero - 1] = C("t")
assert l[zero2 - 1].s == "t"
l[zero - 3] = C("u")
assert l[zero2 - 3].s == "u"
with assertRaises(IndexError):
a[zero + 3] = C("z")
with assertRaises(IndexError):
a[zero - 4] = C("z")
with assertRaises(OverflowError):
a[zero + 2**90] = C("z")
with assertRaises(OverflowError):
a[zero - 2**90] = C("z")

def test_append() -> None:
l = [1, 2]
l.append(10)
Expand Down
Loading