diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst index ef48addaba03e9..3ce20311c42c84 100644 --- a/Doc/library/urllib.parse.rst +++ b/Doc/library/urllib.parse.rst @@ -60,11 +60,16 @@ or on combining URL components into a URL string. *missing_as_none* is true. Not defined component are represented an empty string (by default) or ``None`` if *missing_as_none* is true. - The components are not broken up - into smaller parts (for example, the network location is a single string), and % - escapes are not expanded. The delimiters as shown above are not part of the - result, except for a leading slash in the *path* component, which is retained if - present. For example: + The delimiters as shown above are not part of the result, except for a + leading slash in the *path* component, which is retained if present. + + Additionally, the netloc property is broken down into these additional + attributes added to the returned object: username, password, hostname, and + port. + + Percent-encoded sequences are not decoded. + + For example: .. doctest:: :options: +NORMALIZE_WHITESPACE diff --git a/Lib/test/test_free_threading/test_io.py b/Lib/test/test_free_threading/test_io.py index 742da6f8c788b4..e0bd7e211e7302 100644 --- a/Lib/test/test_free_threading/test_io.py +++ b/Lib/test/test_free_threading/test_io.py @@ -122,6 +122,35 @@ def sizeof(barrier, b, *ignore): class CBytesIOTest(ThreadSafetyMixin, TestCase): ioclass = io.BytesIO + @threading_helper.requires_working_threading() + @threading_helper.reap_threads + def test_concurrent_whole_buffer_read_and_resize(self): + shared = self.ioclass(b"x" * 64) + writers = 2 + readers = 8 + loops = 2000 + barrier = threading.Barrier(writers + readers) + + def writer(): + barrier.wait() + for i in range(loops): + shared.seek(0) + shared.write(b"a" * (64 + (i & 63))) + + def reader(): + barrier.wait() + for _ in range(loops): + shared.seek(0) + shared.read() + shared.seek(0) + shared.peek() + shared.getvalue() + + threads = [threading.Thread(target=writer) for _ in range(writers)] + threads += [threading.Thread(target=reader) for _ in range(readers)] + with threading_helper.start_threads(threads): + pass + class PyBytesIOTest(ThreadSafetyMixin, TestCase): ioclass = pyio.BytesIO diff --git a/Lib/test/test_io/test_memoryio.py b/Lib/test/test_io/test_memoryio.py index 5a93d263458046..f53525d77b7dba 100644 --- a/Lib/test/test_io/test_memoryio.py +++ b/Lib/test/test_io/test_memoryio.py @@ -939,7 +939,10 @@ def test_setstate(self): @support.cpython_only def test_sizeof(self): - basesize = support.calcobjsize('P2n2Pn') + if support.Py_GIL_DISABLED: + basesize = support.calcobjsize('P2n2Pni') + else: + basesize = support.calcobjsize('P2n2Pn') check = self.check_sizeof self.assertEqual(object.__sizeof__(io.BytesIO()), basesize) check(io.BytesIO(), basesize ) diff --git a/Misc/NEWS.d/next/Library/2026-06-18-12-48-03.gh-issue-151640.R4c3Fx.rst b/Misc/NEWS.d/next/Library/2026-06-18-12-48-03.gh-issue-151640.R4c3Fx.rst new file mode 100644 index 00000000000000..ef61a7d27dd07d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-18-12-48-03.gh-issue-151640.R4c3Fx.rst @@ -0,0 +1,3 @@ +Fix a data race in :class:`io.BytesIO` in free-threaded builds when +whole-buffer reads or peeks, or :meth:`~io.BytesIO.getvalue`, share the +internal buffer with concurrent writes. diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c index ffe82150be1b51..3d14ec3f8f94a9 100644 --- a/Modules/_io/bytesio.c +++ b/Modules/_io/bytesio.c @@ -22,6 +22,9 @@ typedef struct { PyObject *dict; PyObject *weakreflist; Py_ssize_t exports; +#ifdef Py_GIL_DISABLED + int buf_shared; +#endif } bytesio; #define bytesio_CAST(op) ((bytesio *)(op)) @@ -71,7 +74,45 @@ check_exports(bytesio *self) return NULL; \ } +#ifdef Py_GIL_DISABLED +#define SHARED_BUF(self) ((self)->buf_shared || !_PyObject_IsUniquelyReferenced((self)->buf)) +#else #define SHARED_BUF(self) (!_PyObject_IsUniquelyReferenced((self)->buf)) +#endif + +static inline void +set_shared_buf(bytesio *self) +{ +#ifdef Py_GIL_DISABLED + self->buf_shared = 1; +#endif +} + +static inline void +clear_shared_buf(bytesio *self) +{ +#ifdef Py_GIL_DISABLED + self->buf_shared = 0; +#endif +} + +static int +resize_unshared_buffer_lock_held(bytesio *self, Py_ssize_t size) +{ + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); + +#ifdef Py_GIL_DISABLED + /* If the internal bytes object escaped via a zero-copy getvalue(), read(), + or peek(), resizing it would mutate an object visible to Python code. + Callers must detach first. */ + assert(!self->buf_shared); +#endif + int ret = _PyBytes_Resize(&self->buf, size); + if (ret == 0) { + clear_shared_buf(self); + } + return ret; +} /* Internal routine to get a line from the buffer of a BytesIO @@ -128,6 +169,7 @@ unshare_buffer_lock_held(bytesio *self, size_t size) memcpy(PyBytes_AS_STRING(new_buf), PyBytes_AS_STRING(self->buf), self->string_size); Py_SETREF(self->buf, new_buf); + clear_shared_buf(self); return 0; } @@ -173,7 +215,7 @@ resize_buffer_lock_held(bytesio *self, size_t size) return -1; } else { - if (_PyBytes_Resize(&self->buf, alloc) < 0) + if (resize_unshared_buffer_lock_held(self, alloc) < 0) return -1; } @@ -381,10 +423,11 @@ _io_BytesIO_getvalue_impl(bytesio *self) return NULL; } else { - if (_PyBytes_Resize(&self->buf, self->string_size) < 0) + if (resize_unshared_buffer_lock_held(self, self->string_size) < 0) return NULL; } } + set_shared_buf(self); return Py_NewRef(self->buf); } @@ -433,6 +476,7 @@ peek_bytes_lock_held(bytesio *self, Py_ssize_t size) if (size > 1 && self->pos == 0 && size == PyBytes_GET_SIZE(self->buf) && FT_ATOMIC_LOAD_SSIZE_RELAXED(self->exports) == 0) { + set_shared_buf(self); return Py_NewRef(self->buf); } @@ -1091,6 +1135,7 @@ _io_BytesIO___init___impl(bytesio *self, PyObject *initvalue) if (initvalue && initvalue != Py_None) { if (PyBytes_CheckExact(initvalue)) { Py_XSETREF(self->buf, Py_NewRef(initvalue)); + clear_shared_buf(self); self->string_size = PyBytes_GET_SIZE(initvalue); } else { diff --git a/Modules/_sre/sre.c b/Modules/_sre/sre.c index 7d03b909226f24..e742a25e4891fc 100644 --- a/Modules/_sre/sre.c +++ b/Modules/_sre/sre.c @@ -738,6 +738,7 @@ state_init(SRE_STATE* state, PatternObject* pattern, PyObject* string, state->charsize = charsize; state->match_all = 0; state->must_advance = 0; + state->save_marks = 0; state->debug = ((pattern->flags & SRE_FLAG_DEBUG) != 0); state->beginning = ptr; diff --git a/Modules/_sre/sre.h b/Modules/_sre/sre.h index 42681c2addf3c2..bec4ae21d1e10f 100644 --- a/Modules/_sre/sre.h +++ b/Modules/_sre/sre.h @@ -97,6 +97,10 @@ typedef struct { int lastmark; int lastindex; const void** mark; + int save_marks; /* if nonzero, save and restore mark values on + backtracking instead of only rewinding the lastmark + index; counts enclosing repeat contexts and + possessive bodies */ /* dynamically allocated stuff */ char* data_stack; size_t data_stack_size; diff --git a/Modules/_sre/sre_lib.h b/Modules/_sre/sre_lib.h index 6e6ae46f05a50f..444cd39d2fed28 100644 --- a/Modules/_sre/sre_lib.h +++ b/Modules/_sre/sre_lib.h @@ -863,7 +863,7 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) /* <0=skip> code ... */ TRACE(("|%p|%p|BRANCH\n", pattern, ptr)); LASTMARK_SAVE(); - if (state->repeat) + if (state->save_marks) MARK_PUSH(ctx->lastmark); for (; pattern[0]; pattern += pattern[0]) { if (pattern[1] == SRE_OP_LITERAL && @@ -878,16 +878,16 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) state->ptr = ptr; DO_JUMP(JUMP_BRANCH, jump_branch, pattern+1); if (ret) { - if (state->repeat) + if (state->save_marks) MARK_POP_DISCARD(ctx->lastmark); RETURN_ON_ERROR(ret); RETURN_SUCCESS; } - if (state->repeat) + if (state->save_marks) MARK_POP_KEEP(ctx->lastmark); LASTMARK_RESTORE(); } - if (state->repeat) + if (state->save_marks) MARK_POP_DISCARD(ctx->lastmark); RETURN_FAILURE; @@ -933,7 +933,7 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) } LASTMARK_SAVE(); - if (state->repeat) + if (state->save_marks) MARK_PUSH(ctx->lastmark); if (pattern[pattern[0]] == SRE_OP_LITERAL) { @@ -952,19 +952,19 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) DO_JUMP(JUMP_REPEAT_ONE_1, jump_repeat_one_1, pattern+pattern[0]); if (ret) { - if (state->repeat) + if (state->save_marks) MARK_POP_DISCARD(ctx->lastmark); RETURN_ON_ERROR(ret); RETURN_SUCCESS; } - if (state->repeat) + if (state->save_marks) MARK_POP_KEEP(ctx->lastmark); LASTMARK_RESTORE(); ptr--; ctx->count--; } - if (state->repeat) + if (state->save_marks) MARK_POP_DISCARD(ctx->lastmark); } else { /* general case */ @@ -973,19 +973,19 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) DO_JUMP(JUMP_REPEAT_ONE_2, jump_repeat_one_2, pattern+pattern[0]); if (ret) { - if (state->repeat) + if (state->save_marks) MARK_POP_DISCARD(ctx->lastmark); RETURN_ON_ERROR(ret); RETURN_SUCCESS; } - if (state->repeat) + if (state->save_marks) MARK_POP_KEEP(ctx->lastmark); LASTMARK_RESTORE(); ptr--; ctx->count--; } - if (state->repeat) + if (state->save_marks) MARK_POP_DISCARD(ctx->lastmark); } RETURN_FAILURE; @@ -1035,7 +1035,7 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) } else { /* general case */ LASTMARK_SAVE(); - if (state->repeat) + if (state->save_marks) MARK_PUSH(ctx->lastmark); while ((Py_ssize_t)pattern[2] == SRE_MAXREPEAT @@ -1044,12 +1044,12 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) DO_JUMP(JUMP_MIN_REPEAT_ONE,jump_min_repeat_one, pattern+pattern[0]); if (ret) { - if (state->repeat) + if (state->save_marks) MARK_POP_DISCARD(ctx->lastmark); RETURN_ON_ERROR(ret); RETURN_SUCCESS; } - if (state->repeat) + if (state->save_marks) MARK_POP_KEEP(ctx->lastmark); LASTMARK_RESTORE(); @@ -1063,7 +1063,7 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) ptr++; ctx->count++; } - if (state->repeat) + if (state->save_marks) MARK_POP_DISCARD(ctx->lastmark); } RETURN_FAILURE; @@ -1140,10 +1140,12 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) ctx->u.rep->prev = state->repeat; ctx->u.rep->last_ptr = NULL; state->repeat = ctx->u.rep; + state->save_marks++; state->ptr = ptr; DO_JUMP(JUMP_REPEAT, jump_repeat, pattern+pattern[0]); state->repeat = ctx->u.rep->prev; + state->save_marks--; repeat_pool_free(state, ctx->u.rep); if (ret) { @@ -1212,8 +1214,10 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) /* cannot match more repeated items here. make sure the tail matches */ state->repeat = ctx->u.rep->prev; + state->save_marks--; DO_JUMP(JUMP_MAX_UNTIL_3, jump_max_until_3, pattern); state->repeat = ctx->u.rep; // restore repeat before return + state->save_marks++; RETURN_ON_SUCCESS(ret); state->ptr = ptr; @@ -1250,22 +1254,26 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) /* see if the tail matches */ state->repeat = ctx->u.rep->prev; + state->save_marks--; LASTMARK_SAVE(); - if (state->repeat) + if (state->save_marks) MARK_PUSH(ctx->lastmark); DO_JUMP(JUMP_MIN_UNTIL_2, jump_min_until_2, pattern); - SRE_REPEAT *repeat_of_tail = state->repeat; + /* save_marks is balanced across the jump, so this equals the + value tested for MARK_PUSH above */ + int pushed = state->save_marks != 0; state->repeat = ctx->u.rep; // restore repeat before return + state->save_marks++; if (ret) { - if (repeat_of_tail) + if (pushed) MARK_POP_DISCARD(ctx->lastmark); RETURN_ON_ERROR(ret); RETURN_SUCCESS; } - if (repeat_of_tail) + if (pushed) MARK_POP(ctx->lastmark); LASTMARK_RESTORE(); @@ -1302,16 +1310,10 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) pointer */ state->ptr = ptr; - /* Set state->repeat to non-NULL */ - ctx->u.rep = repeat_pool_malloc(state); - if (!ctx->u.rep) { - RETURN_ERROR(SRE_ERROR_MEMORY); - } - ctx->u.rep->count = -1; - ctx->u.rep->pattern = NULL; - ctx->u.rep->prev = state->repeat; - ctx->u.rep->last_ptr = NULL; - state->repeat = ctx->u.rep; + /* Capture groups in the body can be revisited on backtracking + between iterations, so their marks must be saved and restored, + as is done inside a repeat. */ + state->save_marks++; /* Initialize Count to 0 */ ctx->count = 0; @@ -1327,9 +1329,7 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) } else { state->ptr = ptr; - /* Restore state->repeat */ - state->repeat = ctx->u.rep->prev; - repeat_pool_free(state, ctx->u.rep); + state->save_marks--; RETURN_FAILURE; } } @@ -1402,9 +1402,7 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) } } - /* Restore state->repeat */ - state->repeat = ctx->u.rep->prev; - repeat_pool_free(state, ctx->u.rep); + state->save_marks--; /* Evaluate Tail */ /* Jump to end of pattern indicated by skip, and then skip @@ -1586,17 +1584,17 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) if ((uintptr_t)(ptr - (SRE_CHAR *)state->beginning) >= pattern[1]) { state->ptr = ptr - pattern[1]; LASTMARK_SAVE(); - if (state->repeat) + if (state->save_marks) MARK_PUSH(ctx->lastmark); DO_JUMP0(JUMP_ASSERT_NOT, jump_assert_not, pattern+2); if (ret) { - if (state->repeat) + if (state->save_marks) MARK_POP_DISCARD(ctx->lastmark); RETURN_ON_ERROR(ret); RETURN_FAILURE; } - if (state->repeat) + if (state->save_marks) MARK_POP(ctx->lastmark); LASTMARK_RESTORE(); }