diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index 836aa91bb0885b..dd9e27ddc27651 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -2046,10 +2046,12 @@ Socket Objects .. method:: sendfile(file, offset=0, count=None) Send a file until EOF is reached by using high-performance - :mod:`os.sendfile` and return the total number of bytes which were sent. + :mod:`os.sendfile` or :c:func:`!TransmitFile` and return the total + number of bytes which were sent. *file* must be a regular file object opened in binary mode. If - :mod:`os.sendfile` is not available (e.g. Windows) or *file* is not a - regular file :meth:`send` will be used instead. *offset* tells from where to + :mod:`os.sendfile` or :c:func:`!TransmitFile` is not available or + *file* is not a regular file :meth:`send` will be used + instead. *offset* tells from where to start reading the file. If specified, *count* is the total number of bytes to transmit as opposed to sending the file until EOF is reached. File position is updated on return or also in case of error in which case @@ -2059,6 +2061,10 @@ Socket Objects .. versionadded:: 3.5 + .. versionchanged:: next + On Windows, :c:func:`!TransmitFile` is now used instead of falling + back to :meth:`send`. + .. method:: set_inheritable(inheritable) Set the :ref:`inheritable flag ` of the socket's file diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index ca80b0a1227588..42ba0131770998 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -243,6 +243,15 @@ shlex (Contributed by Jay Berry in :gh:`148846`.) +socket +------ + +* :meth:`socket.socket.sendfile` now uses the :c:func:`!TransmitFile` system + call on Windows, instead of falling back to a slower + :meth:`~socket.socket.send` loop. + (Contributed by An Long in :gh:`65920`.) + + tkinter ------- diff --git a/Lib/socket.py b/Lib/socket.py index 03c3fe88f15cfe..d4361398a975e1 100644 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -58,6 +58,13 @@ from enum import IntEnum, IntFlag from functools import partial +try: + import _overlapped + import msvcrt +except ImportError: + _overlapped = None + msvcrt = None + try: import errno except ImportError: @@ -467,6 +474,66 @@ def _sendfile_use_send(self, file, offset=0, count=None): if total_sent > 0 and hasattr(file, 'seek'): file.seek(offset + total_sent) + if _overlapped and msvcrt: + def _sendfile_use_transmitfile(self, file, offset=0, count=None): + self._check_sendfile_params(file, offset, count) + timeout = self.gettimeout() + if timeout == 0: + raise ValueError("non-blocking sockets are not supported") + try: + fileno = file.fileno() + except (AttributeError, io.UnsupportedOperation) as err: + raise _GiveupOnSendfile(err) # not a regular file + try: + os.fstat(fileno) + except OSError as err: + raise _GiveupOnSendfile(err) # not a regular file + sock_fileno = self.fileno() + file_handle = msvcrt.get_osfhandle(fileno) + timeout_ms = _overlapped.INFINITE + if timeout is not None: + timeout_ms = int(timeout * 1000) + + max_count = 0xffff_ffff + total_sent = 0 + remaining = count + try: + while True: + chunk_offset = offset + total_sent + offset_low = chunk_offset & 0xffff_ffff + offset_high = (chunk_offset >> 32) & 0xffff_ffff + if remaining is None: + chunk_count = 0 + else: + chunk_count = min(remaining, max_count) + if chunk_count <= 0: + break + + ov = _overlapped.Overlapped() + ov.TransmitFile(sock_fileno, file_handle, offset_low, + offset_high, chunk_count, 0, 0) + try: + sent = ov.getresultex(timeout_ms, False) + except WindowsError as e: + if e.winerror == 258: + raise TimeoutError('timed out') + raise + + total_sent += sent + + if remaining is None: + if sent == 0: + break + else: + remaining -= sent + if sent < chunk_count: + break + + return total_sent + finally: + if total_sent > 0 and hasattr(file, 'seek'): + file.seek(offset + total_sent) + def _check_sendfile_params(self, file, offset, count): if 'b' not in getattr(file, 'mode', 'b'): raise ValueError("file should be opened in binary mode") @@ -484,10 +551,10 @@ def sendfile(self, file, offset=0, count=None): """sendfile(file[, offset[, count]]) -> sent Send a file until EOF is reached by using high-performance - os.sendfile() and return the total number of bytes which - were sent. + os.sendfile() or TransmitFile() and return the total number of + bytes which were sent. *file* must be a regular file object opened in binary mode. - If os.sendfile() is not available (e.g. Windows) or file is + If os.sendfile() or TransmitFile() is not available or file is not a regular file socket.send() will be used instead. *offset* tells from where to start reading the file. If specified, *count* is the total number of bytes to transmit @@ -499,6 +566,11 @@ def sendfile(self, file, offset=0, count=None): Non-blocking sockets are not supported. """ try: + if sys.platform == "win32": + sendfile_use_transmitfile = getattr( + self, "_sendfile_use_transmitfile", None) + if sendfile_use_transmitfile is not None: + return sendfile_use_transmitfile(file, offset, count) return self._sendfile_use_sendfile(file, offset, count) except _GiveupOnSendfile: return self._sendfile_use_send(file, offset, count) diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 542d97e3f886e2..7d2881346d00b0 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -7101,6 +7101,15 @@ def meth_from_sock(self, sock): return getattr(sock, "_sendfile_use_sendfile") +@unittest.skipUnless(sys.platform == "win32", "Windows only test.") +class SendfileUsingTransmitfileTest(SendfileUsingSendTest): + """ + Test the TransmitFile() implementation of socket.sendfile(). + """ + def meth_from_sock(self, sock): + return getattr(sock, "_sendfile_use_transmitfile") + + @unittest.skipUnless(HAVE_SOCKET_ALG, 'AF_ALG required') class LinuxKernelCryptoAPI(unittest.TestCase): # tests for AF_ALG diff --git a/Misc/NEWS.d/next/Windows/2023-11-23-22-25-16.gh-issue-65920.6wWsHN.rst b/Misc/NEWS.d/next/Windows/2023-11-23-22-25-16.gh-issue-65920.6wWsHN.rst new file mode 100644 index 00000000000000..824b191380add9 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2023-11-23-22-25-16.gh-issue-65920.6wWsHN.rst @@ -0,0 +1 @@ +Use :c:func:`!TransmitFile` on Windows to implement :func:`!socket.sendfile`. diff --git a/Modules/clinic/overlapped.c.h b/Modules/clinic/overlapped.c.h index ba41eab15650e8..ea034c43dda108 100644 --- a/Modules/clinic/overlapped.c.h +++ b/Modules/clinic/overlapped.c.h @@ -562,6 +562,46 @@ _overlapped_Overlapped_getresult(PyObject *self, PyObject *const *args, Py_ssize return return_value; } +PyDoc_STRVAR(_overlapped_Overlapped_getresultex__doc__, +"getresultex($self, milliseconds, alertable, /)\n" +"--\n" +"\n" +"Retrieve result of operation, waiting up to milliseconds.\n" +"\n" +"If the operation does not finish within milliseconds, a\n" +"WAIT_TIMEOUT error is raised. If alertable is true the wait can\n" +"also be interrupted to run queued APCs."); + +#define _OVERLAPPED_OVERLAPPED_GETRESULTEX_METHODDEF \ + {"getresultex", _PyCFunction_CAST(_overlapped_Overlapped_getresultex), METH_FASTCALL, _overlapped_Overlapped_getresultex__doc__}, + +static PyObject * +_overlapped_Overlapped_getresultex_impl(OverlappedObject *self, + DWORD milliseconds, BOOL alertable); + +static PyObject * +_overlapped_Overlapped_getresultex(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + DWORD milliseconds; + BOOL alertable; + + if (!_PyArg_CheckPositional("getresultex", nargs, 2, 2)) { + goto exit; + } + if (!_PyLong_UnsignedLong_Converter(args[0], &milliseconds)) { + goto exit; + } + alertable = PyLong_AsInt(args[1]); + if (alertable == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = _overlapped_Overlapped_getresultex_impl((OverlappedObject *)self, milliseconds, alertable); + +exit: + return return_value; +} + PyDoc_STRVAR(_overlapped_Overlapped_ReadFile__doc__, "ReadFile($self, handle, size, /)\n" "--\n" @@ -1243,4 +1283,4 @@ _overlapped_Overlapped_WSARecvFromInto(PyObject *self, PyObject *const *args, Py return return_value; } -/*[clinic end generated code: output=0ecaf45a09539599 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=05e84dc50eddda53 input=a9049054013a1b77]*/ diff --git a/Modules/overlapped.c b/Modules/overlapped.c index 255576cc057cdd..efb428687a1eeb 100644 --- a/Modules/overlapped.c +++ b/Modules/overlapped.c @@ -877,46 +877,14 @@ _overlapped_Overlapped_cancel_impl(OverlappedObject *self) Py_RETURN_NONE; } -/*[clinic input] -_overlapped.Overlapped.getresult - - wait: BOOL(c_default='FALSE') = False - / - -Retrieve result of operation. - -If wait is true then it blocks until the operation is finished. If -wait is false and the operation is still pending then an error is -raised. -[clinic start generated code]*/ - static PyObject * -_overlapped_Overlapped_getresult_impl(OverlappedObject *self, BOOL wait) -/*[clinic end generated code: output=8c9bd04d08994f6c input=852fbd817cbd2b3d]*/ +check_getresult_error(OverlappedObject *self, DWORD transferred) { - DWORD transferred = 0; - BOOL ret; - DWORD err; - PyObject *addr; + PyObject *addr = NULL; PyObject *transferred_obj; + DWORD err = self->error; - if (self->type == TYPE_NONE) { - PyErr_SetString(PyExc_ValueError, "operation not yet attempted"); - return NULL; - } - - if (self->type == TYPE_NOT_STARTED) { - PyErr_SetString(PyExc_ValueError, "operation failed to start"); - return NULL; - } - - Py_BEGIN_ALLOW_THREADS - ret = GetOverlappedResult(self->handle, &self->overlapped, &transferred, - wait); - Py_END_ALLOW_THREADS - - self->error = err = ret ? ERROR_SUCCESS : GetLastError(); - switch (err) { + switch (self->error) { case ERROR_SUCCESS: case ERROR_MORE_DATA: break; @@ -1002,6 +970,88 @@ _overlapped_Overlapped_getresult_impl(OverlappedObject *self, BOOL wait) } } +/*[clinic input] +_overlapped.Overlapped.getresult + + wait: BOOL(c_default='FALSE') = False + / + +Retrieve result of operation. + +If wait is true then it blocks until the operation is finished. If +wait is false and the operation is still pending then an error is +raised. +[clinic start generated code]*/ + +static PyObject * +_overlapped_Overlapped_getresult_impl(OverlappedObject *self, BOOL wait) +/*[clinic end generated code: output=8c9bd04d08994f6c input=852fbd817cbd2b3d]*/ +{ + DWORD transferred = 0; + BOOL ret; + + if (self->type == TYPE_NONE) { + PyErr_SetString(PyExc_ValueError, "operation not yet attempted"); + return NULL; + } + + if (self->type == TYPE_NOT_STARTED) { + PyErr_SetString(PyExc_ValueError, "operation failed to start"); + return NULL; + } + + Py_BEGIN_ALLOW_THREADS + ret = GetOverlappedResult(self->handle, &self->overlapped, &transferred, + wait); + Py_END_ALLOW_THREADS + + self->error = ret ? ERROR_SUCCESS : GetLastError(); + + return check_getresult_error(self, transferred); +} + +/*[clinic input] +_overlapped.Overlapped.getresultex + + milliseconds: DWORD + alertable: BOOL + / + +Retrieve result of operation, waiting up to milliseconds. + +If the operation does not finish within milliseconds, a +WAIT_TIMEOUT error is raised. If alertable is true the wait can +also be interrupted to run queued APCs. +[clinic start generated code]*/ + +static PyObject * +_overlapped_Overlapped_getresultex_impl(OverlappedObject *self, + DWORD milliseconds, BOOL alertable) +/*[clinic end generated code: output=ce0eb6ffb9618e54 input=9891d8ae4afe2b00]*/ +{ + DWORD transferred = 0; + BOOL ret; + + if (self->type == TYPE_NONE) { + PyErr_SetString(PyExc_ValueError, "operation not yet attempted"); + return NULL; + } + + if (self->type == TYPE_NOT_STARTED) { + PyErr_SetString(PyExc_ValueError, "operation failed to start"); + return NULL; + } + + Py_BEGIN_ALLOW_THREADS + ret = GetOverlappedResultEx(self->handle, &self->overlapped, &transferred, + milliseconds, alertable); + Py_END_ALLOW_THREADS + + self->error = ret ? ERROR_SUCCESS : GetLastError(); + + return check_getresult_error(self, transferred); +} + static PyObject * do_ReadFile(OverlappedObject *self, HANDLE handle, char *bufstart, DWORD buflen) @@ -1952,6 +2002,7 @@ _overlapped_Overlapped_WSARecvFromInto_impl(OverlappedObject *self, static PyMethodDef Overlapped_methods[] = { _OVERLAPPED_OVERLAPPED_GETRESULT_METHODDEF + _OVERLAPPED_OVERLAPPED_GETRESULTEX_METHODDEF _OVERLAPPED_OVERLAPPED_CANCEL_METHODDEF _OVERLAPPED_OVERLAPPED_READFILE_METHODDEF _OVERLAPPED_OVERLAPPED_READFILEINTO_METHODDEF