From 3c8c22a185aff52239ef124bcfe3c76e0cc180cd Mon Sep 17 00:00:00 2001 From: Abdullah Masood Date: Wed, 15 Jul 2026 11:05:49 +0500 Subject: [PATCH 1/7] Fix cross-interpreter memory leak in _xidata_release via atomic claim-and-free --- Include/internal/pycore_crossinterp.h | 10 ++ Include/internal/pycore_interp_structs.h | 3 + .../2026-07-15-11-06-00.gh-issue-153713.rst | 1 + Python/crossinterp.c | 103 +++++++++++++++--- Python/pystate.c | 5 + 5 files changed, 105 insertions(+), 17 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2026-07-15-11-06-00.gh-issue-153713.rst diff --git a/Include/internal/pycore_crossinterp.h b/Include/internal/pycore_crossinterp.h index bed966681fa1f0c..669ffb67e2833f4 100644 --- a/Include/internal/pycore_crossinterp.h +++ b/Include/internal/pycore_crossinterp.h @@ -42,6 +42,12 @@ typedef struct _xidata _PyXIData_t; typedef PyObject *(*xid_newobjfunc)(_PyXIData_t *); typedef void (*xid_freefunc)(void *); +typedef enum { + _PyXIData_STATUS_ACTIVE = 0, + _PyXIData_STATUS_CLAIMED_BY_SENDER_TEARDOWN, + _PyXIData_STATUS_RELEASED, +} _PyXIData_status_t; + // _PyXIData_t is similar to Py_buffer as an effectively // opaque struct that holds data outside the object machinery. This // is necessary to pass safely between interpreters in the same process. @@ -82,10 +88,14 @@ struct _xidata { // to PyMem_RawFree (the default if not explicitly set to NULL). // The call will happen with the original interpreter activated. xid_freefunc free; + _Py_atomic_int status; /* one of _PyXIData_status_t */ + struct _xidata *xid_next; + struct _xidata *xid_prev; }; PyAPI_FUNC(_PyXIData_t *) _PyXIData_New(void); PyAPI_FUNC(void) _PyXIData_Free(_PyXIData_t *data); +PyAPI_FUNC(void) _PyXIData_CleanupRegistry(PyInterpreterState *interp); #define _PyXIData_DATA(DATA) ((DATA)->data) #define _PyXIData_OBJ(DATA) ((DATA)->obj) diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index d3efac906aeb933..5b419db17474927 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -1069,6 +1069,9 @@ struct _is { // guards can no longer be created. uintptr_t finalization_guards; + PyMutex xidata_list_mutex; + struct _xidata *xidata_list_head; + /* the initial PyInterpreterState.threads.head */ _PyThreadStateImpl _initial_thread; // _initial_thread should be the last field of PyInterpreterState. diff --git a/Misc/NEWS.d/next/Core and Builtins/2026-07-15-11-06-00.gh-issue-153713.rst b/Misc/NEWS.d/next/Core and Builtins/2026-07-15-11-06-00.gh-issue-153713.rst new file mode 100644 index 000000000000000..716bb09e3b51f11 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2026-07-15-11-06-00.gh-issue-153713.rst @@ -0,0 +1 @@ +Fix a memory leak in :c:func:`_xidata_release` when the originating interpreter of a :c:type:`_PyXIData_t` object is destroyed before the receiving interpreter releases it. The fix implements a lock-free compare-and-swap mechanism to safely coordinate teardown and release across interpreters. diff --git a/Python/crossinterp.c b/Python/crossinterp.c index ed77c1be646e275..bcf5e25639abf1b 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -15,6 +15,59 @@ #include "pycore_runtime.h" // _PyRuntime #include "pycore_setobject.h" // _PySet_NextEntry() #include "pycore_typeobject.h" // _PyStaticType_InitBuiltin() +static void +_PyXIData_Link(PyInterpreterState *interp, _PyXIData_t *xidata) +{ + PyMutex_Lock(&interp->xidata_list_mutex); + xidata->xid_next = interp->xidata_list_head; + xidata->xid_prev = NULL; + if (interp->xidata_list_head != NULL) { + interp->xidata_list_head->xid_prev = xidata; + } + interp->xidata_list_head = xidata; + PyMutex_Unlock(&interp->xidata_list_mutex); +} + +static void +_PyXIData_Unlink(PyInterpreterState *interp, _PyXIData_t *xidata) +{ + PyMutex_Lock(&interp->xidata_list_mutex); + if (xidata->xid_next != NULL) { + xidata->xid_next->xid_prev = xidata->xid_prev; + } + if (xidata->xid_prev != NULL) { + xidata->xid_prev->xid_next = xidata->xid_next; + } + else if (interp->xidata_list_head == xidata) { + interp->xidata_list_head = xidata->xid_next; + } + xidata->xid_next = NULL; + xidata->xid_prev = NULL; + PyMutex_Unlock(&interp->xidata_list_mutex); +} + +void +_PyXIData_CleanupRegistry(PyInterpreterState *interp) +{ + PyMutex_Lock(&interp->xidata_list_mutex); + _PyXIData_t *head = interp->xidata_list_head; + interp->xidata_list_head = NULL; + PyMutex_Unlock(&interp->xidata_list_mutex); + + _PyXIData_t *curr = head; + while (curr != NULL) { + _PyXIData_t *next = curr->xid_next; + int expected = _PyXIData_STATUS_ACTIVE; + if (_Py_atomic_compare_exchange_int(&curr->status, &expected, _PyXIData_STATUS_CLAIMED_BY_SENDER_TEARDOWN)) { + if (curr->free != NULL) { + curr->free(curr->data); + } + curr->data = NULL; + Py_CLEAR(curr->obj); + } + curr = next; + } +} static Py_ssize_t @@ -373,6 +426,13 @@ _PyXIData_Init(_PyXIData_t *xidata, ? PyInterpreterState_GetID(interp) : -1; xidata->new_object = new_object; + + _Py_atomic_store_int(&xidata->status, _PyXIData_STATUS_ACTIVE); + xidata->xid_next = NULL; + xidata->xid_prev = NULL; + if (interp != NULL) { + _PyXIData_Link(interp, xidata); + } } int @@ -1016,6 +1076,10 @@ _PyCode_GetPureScriptXIData(PyThreadState *tstate, PyObject * _PyXIData_NewObject(_PyXIData_t *xidata) { + if (_Py_atomic_load_int(&xidata->status) != _PyXIData_STATUS_ACTIVE) { + PyErr_SetString(PyExc_RuntimeError, "the originating interpreter has been destroyed"); + return NULL; + } return xidata->new_object(xidata); } @@ -1040,26 +1104,31 @@ _xidata_release(_PyXIData_t *xidata, int rawfree) return 0; } - // Switch to the original interpreter. - PyInterpreterState *interp = _PyInterpreterState_LookUpID( - _PyXIData_INTERPID(xidata)); - if (interp == NULL) { - // The interpreter was already destroyed. - // This function shouldn't have been called. - // XXX Someone leaked some memory... - assert(PyErr_Occurred()); - if (rawfree) { - PyMem_RawFree(xidata); + int expected = _PyXIData_STATUS_ACTIVE; + if (_Py_atomic_compare_exchange_int(&xidata->status, &expected, _PyXIData_STATUS_RELEASED)) { + PyInterpreterState *interp = _PyInterpreterState_LookUpID(_PyXIData_INTERPID(xidata)); + if (interp != NULL) { + _PyXIData_Unlink(interp, xidata); + if (rawfree) { + return _Py_CallInInterpreterAndRawFree(interp, _call_clear_xidata, xidata); + } + else { + return _Py_CallInInterpreter(interp, _call_clear_xidata, xidata); + } + } + else { + if (rawfree) { + PyMem_RawFree(xidata); + } + return -1; } - return -1; - } - - // "Release" the data and/or the object. - if (rawfree) { - return _Py_CallInInterpreterAndRawFree(interp, _call_clear_xidata, xidata); } else { - return _Py_CallInInterpreter(interp, _call_clear_xidata, xidata); + // Already claimed by sender teardown. + if (rawfree) { + PyMem_RawFree(xidata); + } + return 0; } } diff --git a/Python/pystate.c b/Python/pystate.c index e90642fa882db72..a33a27b45c323f4 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -8,6 +8,7 @@ #include "pycore_ceval.h" // _PyEval_AcquireLock() #include "pycore_codecs.h" // _PyCodec_Fini() #include "pycore_critical_section.h" // _PyCriticalSection_Resume() +#include "pycore_crossinterp.h" // _PyXIData_CleanupRegistry() #include "pycore_dtoa.h" // _dtoa_state_INIT() #include "pycore_freelist.h" // _PyObject_ClearFreeLists() #include "pycore_initconfig.h" // _PyStatus_OK() @@ -598,6 +599,8 @@ init_interpreter(PyInterpreterState *interp, interp->executor_ptrs = NULL; interp->executor_count = 0; interp->executor_capacity = 0; + interp->xidata_list_mutex = (PyMutex){0}; + interp->xidata_list_head = NULL; interp->executor_deletion_list_head = NULL; interp->executor_creation_counter = JIT_CLEANUP_THRESHOLD; @@ -1066,6 +1069,8 @@ PyInterpreterState_Delete(PyInterpreterState *interp) PyConfig_Clear(&interp->config); + _PyXIData_CleanupRegistry(interp); + free_interpreter(interp); } From 1664301c7d8d8f3835078e6a002d2851931927f8 Mon Sep 17 00:00:00 2001 From: Abdullah Masood Date: Wed, 15 Jul 2026 11:10:45 +0500 Subject: [PATCH 2/7] Fix blurb filename format --- ...-153713.rst => 2026-07-15-11-06-00.gh-issue-153713.abcdef.rst} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Misc/NEWS.d/next/Core and Builtins/{2026-07-15-11-06-00.gh-issue-153713.rst => 2026-07-15-11-06-00.gh-issue-153713.abcdef.rst} (100%) diff --git a/Misc/NEWS.d/next/Core and Builtins/2026-07-15-11-06-00.gh-issue-153713.rst b/Misc/NEWS.d/next/Core and Builtins/2026-07-15-11-06-00.gh-issue-153713.abcdef.rst similarity index 100% rename from Misc/NEWS.d/next/Core and Builtins/2026-07-15-11-06-00.gh-issue-153713.rst rename to Misc/NEWS.d/next/Core and Builtins/2026-07-15-11-06-00.gh-issue-153713.abcdef.rst From 8fa4274f5189ff52afc7b34acc8f13ebd27a5627 Mon Sep 17 00:00:00 2001 From: Abdullah Masood Date: Wed, 15 Jul 2026 11:11:42 +0500 Subject: [PATCH 3/7] Fix MSVC compile error: replace _Py_atomic_int with int --- Include/internal/pycore_crossinterp.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/internal/pycore_crossinterp.h b/Include/internal/pycore_crossinterp.h index 669ffb67e2833f4..da78e0b5b8c79ef 100644 --- a/Include/internal/pycore_crossinterp.h +++ b/Include/internal/pycore_crossinterp.h @@ -88,7 +88,7 @@ struct _xidata { // to PyMem_RawFree (the default if not explicitly set to NULL). // The call will happen with the original interpreter activated. xid_freefunc free; - _Py_atomic_int status; /* one of _PyXIData_status_t */ + int status; /* one of _PyXIData_status_t */ struct _xidata *xid_next; struct _xidata *xid_prev; }; From ed62f3974eed0132c7ab6e15571459eb548bbdf2 Mon Sep 17 00:00:00 2001 From: Abdullah Masood Date: Wed, 15 Jul 2026 11:13:58 +0500 Subject: [PATCH 4/7] Move NEWS blurb to Core_and_Builtins --- .../2026-07-15-11-06-00.gh-issue-153713.abcdef.rst | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Misc/NEWS.d/next/{Core and Builtins => Core_and_Builtins}/2026-07-15-11-06-00.gh-issue-153713.abcdef.rst (100%) diff --git a/Misc/NEWS.d/next/Core and Builtins/2026-07-15-11-06-00.gh-issue-153713.abcdef.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-15-11-06-00.gh-issue-153713.abcdef.rst similarity index 100% rename from Misc/NEWS.d/next/Core and Builtins/2026-07-15-11-06-00.gh-issue-153713.abcdef.rst rename to Misc/NEWS.d/next/Core_and_Builtins/2026-07-15-11-06-00.gh-issue-153713.abcdef.rst From d545695175409e0435081a2ac9a7b00a5e5687ce Mon Sep 17 00:00:00 2001 From: Abdullah Masood Date: Wed, 15 Jul 2026 11:44:36 +0500 Subject: [PATCH 5/7] Fix macOS test failures: resolve stack popped lifetime access race condition and interp lookup race --- Python/crossinterp.c | 11 +++++++---- Python/pystate.c | 4 ++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/Python/crossinterp.c b/Python/crossinterp.c index bcf5e25639abf1b..97a243b409f7dd3 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -57,13 +57,16 @@ _PyXIData_CleanupRegistry(PyInterpreterState *interp) _PyXIData_t *curr = head; while (curr != NULL) { _PyXIData_t *next = curr->xid_next; + void *data = curr->data; + xid_freefunc free_func = curr->free; + PyObject *obj = curr->obj; + int expected = _PyXIData_STATUS_ACTIVE; if (_Py_atomic_compare_exchange_int(&curr->status, &expected, _PyXIData_STATUS_CLAIMED_BY_SENDER_TEARDOWN)) { - if (curr->free != NULL) { - curr->free(curr->data); + if (free_func != NULL) { + free_func(data); } - curr->data = NULL; - Py_CLEAR(curr->obj); + Py_XDECREF(obj); } curr = next; } diff --git a/Python/pystate.c b/Python/pystate.c index a33a27b45c323f4..4ecb1b2ae2e94e5 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1033,6 +1033,8 @@ PyInterpreterState_Delete(PyInterpreterState *interp) zapthreads(interp); + _PyXIData_CleanupRegistry(interp); + // XXX These two calls should be done at the end of clear_interpreter(), // but currently some objects get decref'ed after that. #ifdef Py_REF_DEBUG @@ -1069,8 +1071,6 @@ PyInterpreterState_Delete(PyInterpreterState *interp) PyConfig_Clear(&interp->config); - _PyXIData_CleanupRegistry(interp); - free_interpreter(interp); } From 615037f2af42f4f147e524adda398635d99c79ff Mon Sep 17 00:00:00 2001 From: Abdullah Masood Date: Wed, 15 Jul 2026 11:46:17 +0500 Subject: [PATCH 6/7] Remove trailing whitespace in crossinterp.c --- Python/crossinterp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/crossinterp.c b/Python/crossinterp.c index 97a243b409f7dd3..4d27d650a545313 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -60,7 +60,7 @@ _PyXIData_CleanupRegistry(PyInterpreterState *interp) void *data = curr->data; xid_freefunc free_func = curr->free; PyObject *obj = curr->obj; - + int expected = _PyXIData_STATUS_ACTIVE; if (_Py_atomic_compare_exchange_int(&curr->status, &expected, _PyXIData_STATUS_CLAIMED_BY_SENDER_TEARDOWN)) { if (free_func != NULL) { From 8702c0e85a2d247c2c6bfc876d633ec71f2153d3 Mon Sep 17 00:00:00 2001 From: Abdullah Masood Date: Wed, 15 Jul 2026 11:53:54 +0500 Subject: [PATCH 7/7] Fix Sphinx warnings in NEWS blurb --- .../2026-07-15-11-06-00.gh-issue-153713.abcdef.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-15-11-06-00.gh-issue-153713.abcdef.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-15-11-06-00.gh-issue-153713.abcdef.rst index 716bb09e3b51f11..bdc80a8b5014851 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-15-11-06-00.gh-issue-153713.abcdef.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-15-11-06-00.gh-issue-153713.abcdef.rst @@ -1 +1 @@ -Fix a memory leak in :c:func:`_xidata_release` when the originating interpreter of a :c:type:`_PyXIData_t` object is destroyed before the receiving interpreter releases it. The fix implements a lock-free compare-and-swap mechanism to safely coordinate teardown and release across interpreters. +Fix a memory leak in ``_xidata_release`` when the originating interpreter of a ``_PyXIData_t`` object is destroyed before the receiving interpreter releases it. The fix implements a lock-free compare-and-swap mechanism to safely coordinate teardown and release across interpreters.