From 490df91637b86087b6017d8d104f3f10791cc429 Mon Sep 17 00:00:00 2001 From: AN Long Date: Thu, 23 Nov 2023 20:28:56 +0800 Subject: [PATCH 01/14] add socket._sendfile_use_transmitfile --- Lib/socket.py | 23 +++++++++++++++++++++++ Lib/test/test_socket.py | 8 ++++++++ 2 files changed, 31 insertions(+) diff --git a/Lib/socket.py b/Lib/socket.py index 5f0a1f40e25b94..4fe8f9ff3ccd20 100644 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -55,6 +55,13 @@ import os, sys, io, selectors from enum import IntEnum, IntFlag +try: + import _overlapped + import msvcrt +except ImportError: + _overlapped = None + msvcrt = None + try: import errno except ImportError: @@ -451,6 +458,22 @@ def _sendfile_use_send(self, file, offset=0, count=None): if total_sent > 0 and hasattr(file, 'seek'): file.seek(offset + total_sent) + if _overlapped: + def _sendfile_use_transmitfile(self, file, offset=0, count=None): + self._check_sendfile_params(file, offset, count) + if self.gettimeout() == 0: + raise ValueError("non-blocking sockets are not supported") + ov = _overlapped.Overlapped() + offset_low = offset & 0xffff_ffff + offset_high = (offset >> 32) & 0xffff_ffff + count = count or 0 + ov.TransmitFile(self.fileno(), msvcrt.get_osfhandle(file.fileno()), + offset_low, offset_high, count, 0, 0) + sent = ov.getresult(True) + if sent > 0 and hasattr(file, 'seek'): + file.seek(offset + sent) + return sent + def _check_sendfile_params(self, file, offset, count): if 'b' not in getattr(file, 'mode', 'b'): raise ValueError("file should be opened in binary mode") diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 86701caf05399e..42b234106d300f 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -6421,6 +6421,14 @@ def meth_from_sock(self, sock): return getattr(sock, "_sendfile_use_sendfile") +class SendfileUsingTransmitfileTest(SendfileUsingSendTest): + """ + Test the TransmitFile() implementation of socket.sendfile(). + """ + def meth_from_sock(self, sock): + return getattr(sock, "_sendfile_use_transmitfile") + + @unittest.skipUnless(HAVE_SOCKET_ALG, 'AF_ALG required') class LinuxKernelCryptoAPI(unittest.TestCase): # tests for AF_ALG From f94935df1632812e866c2a27de557433b07bf3ad Mon Sep 17 00:00:00 2001 From: AN Long Date: Thu, 23 Nov 2023 22:21:22 +0800 Subject: [PATCH 02/14] add Overlapped.getresultex to implement timeout in sendfile --- Lib/socket.py | 13 ++++- Modules/clinic/overlapped.c.h | 37 +++++++++++++- Modules/overlapped.c | 92 ++++++++++++++++++++++++++--------- 3 files changed, 116 insertions(+), 26 deletions(-) diff --git a/Lib/socket.py b/Lib/socket.py index 4fe8f9ff3ccd20..b1d6657462056a 100644 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -461,7 +461,8 @@ def _sendfile_use_send(self, file, offset=0, count=None): if _overlapped: def _sendfile_use_transmitfile(self, file, offset=0, count=None): self._check_sendfile_params(file, offset, count) - if self.gettimeout() == 0: + timeout = self.gettimeout() + if timeout == 0: raise ValueError("non-blocking sockets are not supported") ov = _overlapped.Overlapped() offset_low = offset & 0xffff_ffff @@ -469,7 +470,15 @@ def _sendfile_use_transmitfile(self, file, offset=0, count=None): count = count or 0 ov.TransmitFile(self.fileno(), msvcrt.get_osfhandle(file.fileno()), offset_low, offset_high, count, 0, 0) - sent = ov.getresult(True) + timeout_ms = _overlapped.INFINITE + if timeout is not None: + timeout_ms = int(timeout * 1000) + try: + sent = ov.getresultex(timeout_ms, False) + except WindowsError as e: + if e.winerror == 258: + raise TimeoutError('timed out') + raise if sent > 0 and hasattr(file, 'seek'): file.seek(offset + sent) return sent diff --git a/Modules/clinic/overlapped.c.h b/Modules/clinic/overlapped.c.h index 8b285e4a8f0a72..27b328d0b315fa 100644 --- a/Modules/clinic/overlapped.c.h +++ b/Modules/clinic/overlapped.c.h @@ -558,6 +558,41 @@ _overlapped_Overlapped_getresult(OverlappedObject *self, PyObject *const *args, return return_value; } +PyDoc_STRVAR(_overlapped_Overlapped_getresultex__doc__, +"getresultex($self, milliseconds, alertable, /)\n" +"--\n" +"\n"); + +#define _OVERLAPPED_OVERLAPPED_GETRESULTEX_METHODDEF \ + {"getresultex", _PyCFunction_CAST(_overlapped_Overlapped_getresultex), METH_FASTCALL, _overlapped_Overlapped_getresultex__doc__}, + +static PyObject * +_overlapped_Overlapped_getresultex_impl(OverlappedObject *self, + DWORD milliseconds, BOOL alertable); + +static PyObject * +_overlapped_Overlapped_getresultex(OverlappedObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + DWORD milliseconds; + BOOL alertable; + + if (!_PyArg_CheckPositional("getresultex", nargs, 2, 2)) { + goto exit; + } + if (!_PyLong_UnsignedLong_Converter(args[0], &milliseconds)) { + goto exit; + } + alertable = PyLong_AsInt(args[1]); + if (alertable == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = _overlapped_Overlapped_getresultex_impl(self, milliseconds, alertable); + +exit: + return return_value; +} + PyDoc_STRVAR(_overlapped_Overlapped_ReadFile__doc__, "ReadFile($self, handle, size, /)\n" "--\n" @@ -1239,4 +1274,4 @@ _overlapped_Overlapped_WSARecvFromInto(OverlappedObject *self, PyObject *const * return return_value; } -/*[clinic end generated code: output=958cbddbcc355f47 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1f395db21c8adb3f input=a9049054013a1b77]*/ diff --git a/Modules/overlapped.c b/Modules/overlapped.c index fd40e91d0f50c4..542224c3e5407a 100644 --- a/Modules/overlapped.c +++ b/Modules/overlapped.c @@ -862,31 +862,12 @@ is false and the operation is still pending then an error is raised. [clinic start generated code]*/ static PyObject * -_overlapped_Overlapped_getresult_impl(OverlappedObject *self, BOOL wait) -/*[clinic end generated code: output=8c9bd04d08994f6c input=aa5b03e9897ca074]*/ +check_getresult_error(OverlappedObject *self, DWORD transferred) { - DWORD transferred = 0; - BOOL ret; - DWORD err; - PyObject *addr; - - if (self->type == TYPE_NONE) { - PyErr_SetString(PyExc_ValueError, "operation not yet attempted"); - return NULL; - } + PyObject *addr = NULL; + DWORD err = self->error; - if (self->type == TYPE_NOT_STARTED) { - PyErr_SetString(PyExc_ValueError, "operation failed to start"); - return NULL; - } - - Py_BEGIN_ALLOW_THREADS - ret = GetOverlappedResult(self->handle, &self->overlapped, &transferred, - wait); - Py_END_ALLOW_THREADS - - self->error = err = ret ? ERROR_SUCCESS : GetLastError(); - switch (err) { + switch (self->error) { case ERROR_SUCCESS: case ERROR_MORE_DATA: break; @@ -978,6 +959,70 @@ _overlapped_Overlapped_getresult_impl(OverlappedObject *self, BOOL wait) } } +static PyObject * +_overlapped_Overlapped_getresult_impl(OverlappedObject *self, BOOL wait) +/*[clinic end generated code: output=8c9bd04d08994f6c input=aa5b03e9897ca074]*/ +{ + DWORD transferred = 0; + BOOL ret; + + if (self->type == TYPE_NONE) { + PyErr_SetString(PyExc_ValueError, "operation not yet attempted"); + return NULL; + } + + if (self->type == TYPE_NOT_STARTED) { + PyErr_SetString(PyExc_ValueError, "operation failed to start"); + return NULL; + } + + Py_BEGIN_ALLOW_THREADS + ret = GetOverlappedResult(self->handle, &self->overlapped, &transferred, + wait); + Py_END_ALLOW_THREADS + + self->error = ret ? ERROR_SUCCESS : GetLastError(); + + return check_getresult_error(self, transferred); +} + +/*[clinic input] +_overlapped.Overlapped.getresultex + + milliseconds: DWORD + alertable: BOOL + / + +[clinic start generated code]*/ + +static PyObject * +_overlapped_Overlapped_getresultex_impl(OverlappedObject *self, + DWORD milliseconds, BOOL alertable) +/*[clinic end generated code: output=ce0eb6ffb9618e54 input=ef4f4cab49ac1d80]*/ +{ + DWORD transferred = 0; + BOOL ret; + + if (self->type == TYPE_NONE) { + PyErr_SetString(PyExc_ValueError, "operation not yet attempted"); + return NULL; + } + + if (self->type == TYPE_NOT_STARTED) { + PyErr_SetString(PyExc_ValueError, "operation failed to start"); + return NULL; + } + + Py_BEGIN_ALLOW_THREADS + ret = GetOverlappedResultEx(self->handle, &self->overlapped, &transferred, + milliseconds, alertable); + Py_END_ALLOW_THREADS + + self->error = ret ? ERROR_SUCCESS : GetLastError(); + + return check_getresult_error(self, transferred); +} + static PyObject * do_ReadFile(OverlappedObject *self, HANDLE handle, char *bufstart, DWORD buflen) @@ -1927,6 +1972,7 @@ _overlapped_Overlapped_WSARecvFromInto_impl(OverlappedObject *self, static PyMethodDef Overlapped_methods[] = { _OVERLAPPED_OVERLAPPED_GETRESULT_METHODDEF + _OVERLAPPED_OVERLAPPED_GETRESULTEX_METHODDEF _OVERLAPPED_OVERLAPPED_CANCEL_METHODDEF _OVERLAPPED_OVERLAPPED_READFILE_METHODDEF _OVERLAPPED_OVERLAPPED_READFILEINTO_METHODDEF From 87c88faa88c33cdc927868a09025ab23f77c0208 Mon Sep 17 00:00:00 2001 From: AN Long Date: Thu, 23 Nov 2023 22:26:20 +0800 Subject: [PATCH 03/14] using transmitfile for sendfile on windows --- Lib/socket.py | 2 ++ .../next/Windows/2023-11-23-22-25-16.gh-issue-65920.6wWsHN.rst | 1 + 2 files changed, 3 insertions(+) create mode 100644 Misc/NEWS.d/next/Windows/2023-11-23-22-25-16.gh-issue-65920.6wWsHN.rst diff --git a/Lib/socket.py b/Lib/socket.py index b1d6657462056a..431cbb39e6db42 100644 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -514,6 +514,8 @@ def sendfile(self, file, offset=0, count=None): The socket must be of SOCK_STREAM type. Non-blocking sockets are not supported. """ + if sys.platform == "win32": + return self._sendfile_use_transmitfile(file, offset, count) try: return self._sendfile_use_sendfile(file, offset, count) except _GiveupOnSendfile: diff --git a/Misc/NEWS.d/next/Windows/2023-11-23-22-25-16.gh-issue-65920.6wWsHN.rst b/Misc/NEWS.d/next/Windows/2023-11-23-22-25-16.gh-issue-65920.6wWsHN.rst new file mode 100644 index 00000000000000..ff0923890eaff2 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2023-11-23-22-25-16.gh-issue-65920.6wWsHN.rst @@ -0,0 +1 @@ +Use ``TransmitFile`` on Windows to implement ``socket.sendfile``. From 3f7259fa7dbba7bc81e076f9e7da6ac4d65e0b22 Mon Sep 17 00:00:00 2001 From: AN Long Date: Thu, 23 Nov 2023 22:50:37 +0800 Subject: [PATCH 04/14] fix tests --- Lib/socket.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/Lib/socket.py b/Lib/socket.py index 431cbb39e6db42..ab37d28ddf1179 100644 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -468,7 +468,15 @@ def _sendfile_use_transmitfile(self, file, offset=0, count=None): offset_low = offset & 0xffff_ffff offset_high = (offset >> 32) & 0xffff_ffff count = count or 0 - ov.TransmitFile(self.fileno(), msvcrt.get_osfhandle(file.fileno()), + try: + fileno = file.fileno() + except (AttributeError, io.UnsupportedOperation) as err: + raise _GiveupOnSendfile(err) # not a regular file + try: + os.fstat(fileno) + except OSError as err: + raise _GiveupOnSendfile(err) # not a regular file + ov.TransmitFile(self.fileno(), msvcrt.get_osfhandle(fileno), offset_low, offset_high, count, 0, 0) timeout_ms = _overlapped.INFINITE if timeout is not None: @@ -514,9 +522,9 @@ def sendfile(self, file, offset=0, count=None): The socket must be of SOCK_STREAM type. Non-blocking sockets are not supported. """ - if sys.platform == "win32": - return self._sendfile_use_transmitfile(file, offset, count) try: + if sys.platform == "win32": + return self._sendfile_use_transmitfile(file, offset, count) return self._sendfile_use_sendfile(file, offset, count) except _GiveupOnSendfile: return self._sendfile_use_send(file, offset, count) From 619ef2205fdaf69b5c50bc65dd0203d86faf907c Mon Sep 17 00:00:00 2001 From: AN Long Date: Thu, 23 Nov 2023 22:52:01 +0800 Subject: [PATCH 05/14] skip tests on non windows platforms --- Lib/test/test_socket.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 42b234106d300f..28f34e2ec0173e 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -6421,6 +6421,7 @@ def meth_from_sock(self, sock): return getattr(sock, "_sendfile_use_sendfile") +@unittest.skipUnless(sys.platform == "win32", "Windows only test.") class SendfileUsingTransmitfileTest(SendfileUsingSendTest): """ Test the TransmitFile() implementation of socket.sendfile(). From 19dde50495b71c22ffb1566f645966c74f8eacca Mon Sep 17 00:00:00 2001 From: AN Long Date: Thu, 23 Nov 2023 23:02:07 +0800 Subject: [PATCH 06/14] fix generated file --- Modules/overlapped.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Modules/overlapped.c b/Modules/overlapped.c index 542224c3e5407a..f64b2c1c2cfb8d 100644 --- a/Modules/overlapped.c +++ b/Modules/overlapped.c @@ -849,18 +849,6 @@ _overlapped_Overlapped_cancel_impl(OverlappedObject *self) Py_RETURN_NONE; } -/*[clinic input] -_overlapped.Overlapped.getresult - - wait: BOOL(c_default='FALSE') = False - / - -Retrieve result of operation. - -If wait is true then it blocks until the operation is finished. If wait -is false and the operation is still pending then an error is raised. -[clinic start generated code]*/ - static PyObject * check_getresult_error(OverlappedObject *self, DWORD transferred) { @@ -959,6 +947,18 @@ check_getresult_error(OverlappedObject *self, DWORD transferred) } } +/*[clinic input] +_overlapped.Overlapped.getresult + + wait: BOOL(c_default='FALSE') = False + / + +Retrieve result of operation. + +If wait is true then it blocks until the operation is finished. If wait +is false and the operation is still pending then an error is raised. +[clinic start generated code]*/ + static PyObject * _overlapped_Overlapped_getresult_impl(OverlappedObject *self, BOOL wait) /*[clinic end generated code: output=8c9bd04d08994f6c input=aa5b03e9897ca074]*/ From 77deb6e7156800ff41ac56e6bbf09cf00788c6f7 Mon Sep 17 00:00:00 2001 From: AN Long Date: Sun, 7 Jan 2024 16:12:21 +0800 Subject: [PATCH 07/14] Update Misc/NEWS.d/next/Windows/2023-11-23-22-25-16.gh-issue-65920.6wWsHN.rst Co-authored-by: Erlend E. Aasland --- .../next/Windows/2023-11-23-22-25-16.gh-issue-65920.6wWsHN.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Windows/2023-11-23-22-25-16.gh-issue-65920.6wWsHN.rst b/Misc/NEWS.d/next/Windows/2023-11-23-22-25-16.gh-issue-65920.6wWsHN.rst index ff0923890eaff2..890f816013f6f7 100644 --- a/Misc/NEWS.d/next/Windows/2023-11-23-22-25-16.gh-issue-65920.6wWsHN.rst +++ b/Misc/NEWS.d/next/Windows/2023-11-23-22-25-16.gh-issue-65920.6wWsHN.rst @@ -1 +1 @@ -Use ``TransmitFile`` on Windows to implement ``socket.sendfile``. +Use :c:func:`!TransmitFile` on Windows to implement :func:`socket.sendfile`. From 47eb386d7d7bb688eeecb99e4507a714b496a976 Mon Sep 17 00:00:00 2001 From: AN Long Date: Sun, 7 Jan 2024 16:13:48 +0800 Subject: [PATCH 08/14] Update Lib/socket.py --- Lib/socket.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/socket.py b/Lib/socket.py index ab37d28ddf1179..fbdd179e53ccc1 100644 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -458,7 +458,7 @@ def _sendfile_use_send(self, file, offset=0, count=None): if total_sent > 0 and hasattr(file, 'seek'): file.seek(offset + total_sent) - if _overlapped: + if _overlapped and msvcrt: def _sendfile_use_transmitfile(self, file, offset=0, count=None): self._check_sendfile_params(file, offset, count) timeout = self.gettimeout() From 048f6a14131a70c3b85a6dfb861836e26fae7f54 Mon Sep 17 00:00:00 2001 From: AN Long Date: Sun, 1 Mar 2026 18:03:56 +0900 Subject: [PATCH 09/14] Call TransmitFile in chunk --- Lib/socket.py | 55 +++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 40 insertions(+), 15 deletions(-) diff --git a/Lib/socket.py b/Lib/socket.py index c8de9896131d53..af82a5eeb8e6d9 100644 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -480,10 +480,6 @@ def _sendfile_use_transmitfile(self, file, offset=0, count=None): timeout = self.gettimeout() if timeout == 0: raise ValueError("non-blocking sockets are not supported") - ov = _overlapped.Overlapped() - offset_low = offset & 0xffff_ffff - offset_high = (offset >> 32) & 0xffff_ffff - count = count or 0 try: fileno = file.fileno() except (AttributeError, io.UnsupportedOperation) as err: @@ -492,20 +488,49 @@ def _sendfile_use_transmitfile(self, file, offset=0, count=None): os.fstat(fileno) except OSError as err: raise _GiveupOnSendfile(err) # not a regular file - ov.TransmitFile(self.fileno(), msvcrt.get_osfhandle(fileno), - offset_low, offset_high, count, 0, 0) + sock_fileno = self.fileno() + file_handle = msvcrt.get_osfhandle(fileno) timeout_ms = _overlapped.INFINITE if timeout is not None: timeout_ms = int(timeout * 1000) - try: - sent = ov.getresultex(timeout_ms, False) - except WindowsError as e: - if e.winerror == 258: - raise TimeoutError('timed out') - raise - if sent > 0 and hasattr(file, 'seek'): - file.seek(offset + sent) - return sent + + max_count = 0xffff_ffff + total_sent = 0 + remaining = count + while True: + chunk_offset = offset + total_sent + offset_low = chunk_offset & 0xffff_ffff + offset_high = (chunk_offset >> 32) & 0xffff_ffff + if remaining is None: + chunk_count = 0 + else: + chunk_count = min(remaining, max_count) + if chunk_count <= 0: + break + + ov = _overlapped.Overlapped() + ov.TransmitFile(sock_fileno, file_handle, offset_low, + offset_high, chunk_count, 0, 0) + try: + sent = ov.getresultex(timeout_ms, False) + except WindowsError as e: + if e.winerror == 258: + raise TimeoutError('timed out') + raise + + total_sent += sent + + if remaining is None: + if sent == 0: + break + else: + remaining -= sent + if sent < chunk_count: + break + + if total_sent > 0 and hasattr(file, 'seek'): + file.seek(offset + total_sent) + return total_sent def _check_sendfile_params(self, file, offset, count): if 'b' not in getattr(file, 'mode', 'b'): From 6edf90fe572e68794177d379e82a2934b75a5940 Mon Sep 17 00:00:00 2001 From: AN Long Date: Sun, 1 Mar 2026 18:14:43 +0900 Subject: [PATCH 10/14] Fix news entry --- .../next/Windows/2023-11-23-22-25-16.gh-issue-65920.6wWsHN.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Windows/2023-11-23-22-25-16.gh-issue-65920.6wWsHN.rst b/Misc/NEWS.d/next/Windows/2023-11-23-22-25-16.gh-issue-65920.6wWsHN.rst index 890f816013f6f7..824b191380add9 100644 --- a/Misc/NEWS.d/next/Windows/2023-11-23-22-25-16.gh-issue-65920.6wWsHN.rst +++ b/Misc/NEWS.d/next/Windows/2023-11-23-22-25-16.gh-issue-65920.6wWsHN.rst @@ -1 +1 @@ -Use :c:func:`!TransmitFile` on Windows to implement :func:`socket.sendfile`. +Use :c:func:`!TransmitFile` on Windows to implement :func:`!socket.sendfile`. From f2de5026a8536f7ad31ab02217cede9cfc532f02 Mon Sep 17 00:00:00 2001 From: AN Long Date: Sun, 1 Mar 2026 18:34:21 +0900 Subject: [PATCH 11/14] Fallback if _overlapped is not usage --- Lib/socket.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Lib/socket.py b/Lib/socket.py index af82a5eeb8e6d9..7c0940e3576760 100644 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -565,7 +565,10 @@ def sendfile(self, file, offset=0, count=None): """ try: if sys.platform == "win32": - return self._sendfile_use_transmitfile(file, offset, count) + sendfile_use_transmitfile = getattr(self, + "_sendfile_use_transmitfile") + if sendfile_use_transmitfile: + return sendfile_use_transmitfile(file, offset, count) return self._sendfile_use_sendfile(file, offset, count) except _GiveupOnSendfile: return self._sendfile_use_send(file, offset, count) From deee263ac2e48c01977ee98032c2f939af73659d Mon Sep 17 00:00:00 2001 From: AN Long Date: Sun, 1 Mar 2026 18:35:58 +0900 Subject: [PATCH 12/14] Update generated files --- Modules/clinic/overlapped.c.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Modules/clinic/overlapped.c.h b/Modules/clinic/overlapped.c.h index 90743a1b0b6c19..dfc38a1625ccef 100644 --- a/Modules/clinic/overlapped.c.h +++ b/Modules/clinic/overlapped.c.h @@ -574,7 +574,7 @@ _overlapped_Overlapped_getresultex_impl(OverlappedObject *self, DWORD milliseconds, BOOL alertable); static PyObject * -_overlapped_Overlapped_getresultex(OverlappedObject *self, PyObject *const *args, Py_ssize_t nargs) +_overlapped_Overlapped_getresultex(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; DWORD milliseconds; @@ -590,7 +590,7 @@ _overlapped_Overlapped_getresultex(OverlappedObject *self, PyObject *const *args if (alertable == -1 && PyErr_Occurred()) { goto exit; } - return_value = _overlapped_Overlapped_getresultex_impl(self, milliseconds, alertable); + return_value = _overlapped_Overlapped_getresultex_impl((OverlappedObject *)self, milliseconds, alertable); exit: return return_value; @@ -1277,4 +1277,4 @@ _overlapped_Overlapped_WSARecvFromInto(PyObject *self, PyObject *const *args, Py return return_value; } -/*[clinic end generated code: output=3e4cb2b55342cd96 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=3b2367cc36de7750 input=a9049054013a1b77]*/ From 63d384b1f05efcc5d131a6323a3e303767b236bb Mon Sep 17 00:00:00 2001 From: An Long Date: Sat, 27 Jun 2026 21:59:47 +0900 Subject: [PATCH 13/14] Fix file position on error and fallback in socket.sendfile --- Lib/socket.py | 68 ++++++++++++++++++++++++++------------------------- 1 file changed, 35 insertions(+), 33 deletions(-) diff --git a/Lib/socket.py b/Lib/socket.py index 5c64cc52d42f51..390393dbf50557 100644 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -497,40 +497,42 @@ def _sendfile_use_transmitfile(self, file, offset=0, count=None): max_count = 0xffff_ffff total_sent = 0 remaining = count - while True: - chunk_offset = offset + total_sent - offset_low = chunk_offset & 0xffff_ffff - offset_high = (chunk_offset >> 32) & 0xffff_ffff - if remaining is None: - chunk_count = 0 - else: - chunk_count = min(remaining, max_count) - if chunk_count <= 0: - break + try: + while True: + chunk_offset = offset + total_sent + offset_low = chunk_offset & 0xffff_ffff + offset_high = (chunk_offset >> 32) & 0xffff_ffff + if remaining is None: + chunk_count = 0 + else: + chunk_count = min(remaining, max_count) + if chunk_count <= 0: + break - ov = _overlapped.Overlapped() - ov.TransmitFile(sock_fileno, file_handle, offset_low, - offset_high, chunk_count, 0, 0) - try: - sent = ov.getresultex(timeout_ms, False) - except WindowsError as e: - if e.winerror == 258: - raise TimeoutError('timed out') - raise + ov = _overlapped.Overlapped() + ov.TransmitFile(sock_fileno, file_handle, offset_low, + offset_high, chunk_count, 0, 0) + try: + sent = ov.getresultex(timeout_ms, False) + except WindowsError as e: + if e.winerror == 258: + raise TimeoutError('timed out') + raise - total_sent += sent + total_sent += sent - if remaining is None: - if sent == 0: - break - else: - remaining -= sent - if sent < chunk_count: - break + if remaining is None: + if sent == 0: + break + else: + remaining -= sent + if sent < chunk_count: + break - if total_sent > 0 and hasattr(file, 'seek'): - file.seek(offset + total_sent) - return total_sent + return total_sent + finally: + if total_sent > 0 and hasattr(file, 'seek'): + file.seek(offset + total_sent) def _check_sendfile_params(self, file, offset, count): if 'b' not in getattr(file, 'mode', 'b'): @@ -565,9 +567,9 @@ def sendfile(self, file, offset=0, count=None): """ try: if sys.platform == "win32": - sendfile_use_transmitfile = getattr(self, - "_sendfile_use_transmitfile") - if sendfile_use_transmitfile: + sendfile_use_transmitfile = getattr( + self, "_sendfile_use_transmitfile", None) + if sendfile_use_transmitfile is not None: return sendfile_use_transmitfile(file, offset, count) return self._sendfile_use_sendfile(file, offset, count) except _GiveupOnSendfile: From 76e9bc8391297e226351ac536b6f6bd2b1072805 Mon Sep 17 00:00:00 2001 From: An Long Date: Sat, 27 Jun 2026 22:59:51 +0900 Subject: [PATCH 14/14] Update sendfile docs and add getresultex docstring --- Doc/library/socket.rst | 12 +++++++++--- Doc/whatsnew/3.16.rst | 9 +++++++++ Lib/socket.py | 6 +++--- Modules/clinic/overlapped.c.h | 9 +++++++-- Modules/overlapped.c | 7 ++++++- 5 files changed, 34 insertions(+), 9 deletions(-) diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index 836aa91bb0885b..dd9e27ddc27651 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -2046,10 +2046,12 @@ Socket Objects .. method:: sendfile(file, offset=0, count=None) Send a file until EOF is reached by using high-performance - :mod:`os.sendfile` and return the total number of bytes which were sent. + :mod:`os.sendfile` or :c:func:`!TransmitFile` and return the total + number of bytes which were sent. *file* must be a regular file object opened in binary mode. If - :mod:`os.sendfile` is not available (e.g. Windows) or *file* is not a - regular file :meth:`send` will be used instead. *offset* tells from where to + :mod:`os.sendfile` or :c:func:`!TransmitFile` is not available or + *file* is not a regular file :meth:`send` will be used + instead. *offset* tells from where to start reading the file. If specified, *count* is the total number of bytes to transmit as opposed to sending the file until EOF is reached. File position is updated on return or also in case of error in which case @@ -2059,6 +2061,10 @@ Socket Objects .. versionadded:: 3.5 + .. versionchanged:: next + On Windows, :c:func:`!TransmitFile` is now used instead of falling + back to :meth:`send`. + .. method:: set_inheritable(inheritable) Set the :ref:`inheritable flag ` of the socket's file diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index ca80b0a1227588..42ba0131770998 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -243,6 +243,15 @@ shlex (Contributed by Jay Berry in :gh:`148846`.) +socket +------ + +* :meth:`socket.socket.sendfile` now uses the :c:func:`!TransmitFile` system + call on Windows, instead of falling back to a slower + :meth:`~socket.socket.send` loop. + (Contributed by An Long in :gh:`65920`.) + + tkinter ------- diff --git a/Lib/socket.py b/Lib/socket.py index 390393dbf50557..d4361398a975e1 100644 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -551,10 +551,10 @@ def sendfile(self, file, offset=0, count=None): """sendfile(file[, offset[, count]]) -> sent Send a file until EOF is reached by using high-performance - os.sendfile() and return the total number of bytes which - were sent. + os.sendfile() or TransmitFile() and return the total number of + bytes which were sent. *file* must be a regular file object opened in binary mode. - If os.sendfile() is not available (e.g. Windows) or file is + If os.sendfile() or TransmitFile() is not available or file is not a regular file socket.send() will be used instead. *offset* tells from where to start reading the file. If specified, *count* is the total number of bytes to transmit diff --git a/Modules/clinic/overlapped.c.h b/Modules/clinic/overlapped.c.h index 9dcd43ae716e16..ea034c43dda108 100644 --- a/Modules/clinic/overlapped.c.h +++ b/Modules/clinic/overlapped.c.h @@ -565,7 +565,12 @@ _overlapped_Overlapped_getresult(PyObject *self, PyObject *const *args, Py_ssize PyDoc_STRVAR(_overlapped_Overlapped_getresultex__doc__, "getresultex($self, milliseconds, alertable, /)\n" "--\n" -"\n"); +"\n" +"Retrieve result of operation, waiting up to milliseconds.\n" +"\n" +"If the operation does not finish within milliseconds, a\n" +"WAIT_TIMEOUT error is raised. If alertable is true the wait can\n" +"also be interrupted to run queued APCs."); #define _OVERLAPPED_OVERLAPPED_GETRESULTEX_METHODDEF \ {"getresultex", _PyCFunction_CAST(_overlapped_Overlapped_getresultex), METH_FASTCALL, _overlapped_Overlapped_getresultex__doc__}, @@ -1278,4 +1283,4 @@ _overlapped_Overlapped_WSARecvFromInto(PyObject *self, PyObject *const *args, Py return return_value; } -/*[clinic end generated code: output=90125ab16e16d8be input=a9049054013a1b77]*/ +/*[clinic end generated code: output=05e84dc50eddda53 input=a9049054013a1b77]*/ diff --git a/Modules/overlapped.c b/Modules/overlapped.c index e6e333ed1480bb..efb428687a1eeb 100644 --- a/Modules/overlapped.c +++ b/Modules/overlapped.c @@ -1017,12 +1017,17 @@ _overlapped.Overlapped.getresultex alertable: BOOL / +Retrieve result of operation, waiting up to milliseconds. + +If the operation does not finish within milliseconds, a +WAIT_TIMEOUT error is raised. If alertable is true the wait can +also be interrupted to run queued APCs. [clinic start generated code]*/ static PyObject * _overlapped_Overlapped_getresultex_impl(OverlappedObject *self, DWORD milliseconds, BOOL alertable) -/*[clinic end generated code: output=ce0eb6ffb9618e54 input=ef4f4cab49ac1d80]*/ +/*[clinic end generated code: output=ce0eb6ffb9618e54 input=9891d8ae4afe2b00]*/ { DWORD transferred = 0; BOOL ret;