diff --git a/Include/internal/pycore_crossinterp.h b/Include/internal/pycore_crossinterp.h index bed966681fa1f0..da78e0b5b8c79e 100644 --- a/Include/internal/pycore_crossinterp.h +++ b/Include/internal/pycore_crossinterp.h @@ -42,6 +42,12 @@ typedef struct _xidata _PyXIData_t; typedef PyObject *(*xid_newobjfunc)(_PyXIData_t *); typedef void (*xid_freefunc)(void *); +typedef enum { + _PyXIData_STATUS_ACTIVE = 0, + _PyXIData_STATUS_CLAIMED_BY_SENDER_TEARDOWN, + _PyXIData_STATUS_RELEASED, +} _PyXIData_status_t; + // _PyXIData_t is similar to Py_buffer as an effectively // opaque struct that holds data outside the object machinery. This // is necessary to pass safely between interpreters in the same process. @@ -82,10 +88,14 @@ struct _xidata { // to PyMem_RawFree (the default if not explicitly set to NULL). // The call will happen with the original interpreter activated. xid_freefunc free; + int status; /* one of _PyXIData_status_t */ + struct _xidata *xid_next; + struct _xidata *xid_prev; }; PyAPI_FUNC(_PyXIData_t *) _PyXIData_New(void); PyAPI_FUNC(void) _PyXIData_Free(_PyXIData_t *data); +PyAPI_FUNC(void) _PyXIData_CleanupRegistry(PyInterpreterState *interp); #define _PyXIData_DATA(DATA) ((DATA)->data) #define _PyXIData_OBJ(DATA) ((DATA)->obj) diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index d3efac906aeb93..5b419db1747492 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -1069,6 +1069,9 @@ struct _is { // guards can no longer be created. uintptr_t finalization_guards; + PyMutex xidata_list_mutex; + struct _xidata *xidata_list_head; + /* the initial PyInterpreterState.threads.head */ _PyThreadStateImpl _initial_thread; // _initial_thread should be the last field of PyInterpreterState. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-15-11-06-00.gh-issue-153713.abcdef.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-15-11-06-00.gh-issue-153713.abcdef.rst new file mode 100644 index 00000000000000..bdc80a8b501485 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-15-11-06-00.gh-issue-153713.abcdef.rst @@ -0,0 +1 @@ +Fix a memory leak in ``_xidata_release`` when the originating interpreter of a ``_PyXIData_t`` object is destroyed before the receiving interpreter releases it. The fix implements a lock-free compare-and-swap mechanism to safely coordinate teardown and release across interpreters. diff --git a/Python/crossinterp.c b/Python/crossinterp.c index ed77c1be646e27..4d27d650a54531 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -15,6 +15,62 @@ #include "pycore_runtime.h" // _PyRuntime #include "pycore_setobject.h" // _PySet_NextEntry() #include "pycore_typeobject.h" // _PyStaticType_InitBuiltin() +static void +_PyXIData_Link(PyInterpreterState *interp, _PyXIData_t *xidata) +{ + PyMutex_Lock(&interp->xidata_list_mutex); + xidata->xid_next = interp->xidata_list_head; + xidata->xid_prev = NULL; + if (interp->xidata_list_head != NULL) { + interp->xidata_list_head->xid_prev = xidata; + } + interp->xidata_list_head = xidata; + PyMutex_Unlock(&interp->xidata_list_mutex); +} + +static void +_PyXIData_Unlink(PyInterpreterState *interp, _PyXIData_t *xidata) +{ + PyMutex_Lock(&interp->xidata_list_mutex); + if (xidata->xid_next != NULL) { + xidata->xid_next->xid_prev = xidata->xid_prev; + } + if (xidata->xid_prev != NULL) { + xidata->xid_prev->xid_next = xidata->xid_next; + } + else if (interp->xidata_list_head == xidata) { + interp->xidata_list_head = xidata->xid_next; + } + xidata->xid_next = NULL; + xidata->xid_prev = NULL; + PyMutex_Unlock(&interp->xidata_list_mutex); +} + +void +_PyXIData_CleanupRegistry(PyInterpreterState *interp) +{ + PyMutex_Lock(&interp->xidata_list_mutex); + _PyXIData_t *head = interp->xidata_list_head; + interp->xidata_list_head = NULL; + PyMutex_Unlock(&interp->xidata_list_mutex); + + _PyXIData_t *curr = head; + while (curr != NULL) { + _PyXIData_t *next = curr->xid_next; + void *data = curr->data; + xid_freefunc free_func = curr->free; + PyObject *obj = curr->obj; + + int expected = _PyXIData_STATUS_ACTIVE; + if (_Py_atomic_compare_exchange_int(&curr->status, &expected, _PyXIData_STATUS_CLAIMED_BY_SENDER_TEARDOWN)) { + if (free_func != NULL) { + free_func(data); + } + Py_XDECREF(obj); + } + curr = next; + } +} static Py_ssize_t @@ -373,6 +429,13 @@ _PyXIData_Init(_PyXIData_t *xidata, ? PyInterpreterState_GetID(interp) : -1; xidata->new_object = new_object; + + _Py_atomic_store_int(&xidata->status, _PyXIData_STATUS_ACTIVE); + xidata->xid_next = NULL; + xidata->xid_prev = NULL; + if (interp != NULL) { + _PyXIData_Link(interp, xidata); + } } int @@ -1016,6 +1079,10 @@ _PyCode_GetPureScriptXIData(PyThreadState *tstate, PyObject * _PyXIData_NewObject(_PyXIData_t *xidata) { + if (_Py_atomic_load_int(&xidata->status) != _PyXIData_STATUS_ACTIVE) { + PyErr_SetString(PyExc_RuntimeError, "the originating interpreter has been destroyed"); + return NULL; + } return xidata->new_object(xidata); } @@ -1040,26 +1107,31 @@ _xidata_release(_PyXIData_t *xidata, int rawfree) return 0; } - // Switch to the original interpreter. - PyInterpreterState *interp = _PyInterpreterState_LookUpID( - _PyXIData_INTERPID(xidata)); - if (interp == NULL) { - // The interpreter was already destroyed. - // This function shouldn't have been called. - // XXX Someone leaked some memory... - assert(PyErr_Occurred()); - if (rawfree) { - PyMem_RawFree(xidata); + int expected = _PyXIData_STATUS_ACTIVE; + if (_Py_atomic_compare_exchange_int(&xidata->status, &expected, _PyXIData_STATUS_RELEASED)) { + PyInterpreterState *interp = _PyInterpreterState_LookUpID(_PyXIData_INTERPID(xidata)); + if (interp != NULL) { + _PyXIData_Unlink(interp, xidata); + if (rawfree) { + return _Py_CallInInterpreterAndRawFree(interp, _call_clear_xidata, xidata); + } + else { + return _Py_CallInInterpreter(interp, _call_clear_xidata, xidata); + } + } + else { + if (rawfree) { + PyMem_RawFree(xidata); + } + return -1; } - return -1; - } - - // "Release" the data and/or the object. - if (rawfree) { - return _Py_CallInInterpreterAndRawFree(interp, _call_clear_xidata, xidata); } else { - return _Py_CallInInterpreter(interp, _call_clear_xidata, xidata); + // Already claimed by sender teardown. + if (rawfree) { + PyMem_RawFree(xidata); + } + return 0; } } diff --git a/Python/pystate.c b/Python/pystate.c index e90642fa882db7..4ecb1b2ae2e94e 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -8,6 +8,7 @@ #include "pycore_ceval.h" // _PyEval_AcquireLock() #include "pycore_codecs.h" // _PyCodec_Fini() #include "pycore_critical_section.h" // _PyCriticalSection_Resume() +#include "pycore_crossinterp.h" // _PyXIData_CleanupRegistry() #include "pycore_dtoa.h" // _dtoa_state_INIT() #include "pycore_freelist.h" // _PyObject_ClearFreeLists() #include "pycore_initconfig.h" // _PyStatus_OK() @@ -598,6 +599,8 @@ init_interpreter(PyInterpreterState *interp, interp->executor_ptrs = NULL; interp->executor_count = 0; interp->executor_capacity = 0; + interp->xidata_list_mutex = (PyMutex){0}; + interp->xidata_list_head = NULL; interp->executor_deletion_list_head = NULL; interp->executor_creation_counter = JIT_CLEANUP_THRESHOLD; @@ -1030,6 +1033,8 @@ PyInterpreterState_Delete(PyInterpreterState *interp) zapthreads(interp); + _PyXIData_CleanupRegistry(interp); + // XXX These two calls should be done at the end of clear_interpreter(), // but currently some objects get decref'ed after that. #ifdef Py_REF_DEBUG