gh-65920: Implement socket.sendfile with TransmitFile on Windows

Doc/library/socket.rst

@@ -2046,10 +2046,12 @@ Socket Objects
    .. method:: sendfile(file, offset=0, count=None)
 
       Send a file until EOF is reached by using high-performance
-      :mod:`os.sendfile` and return the total number of bytes which were sent.
+      :mod:`os.sendfile` or :c:func:`!TransmitFile` and return the total
+      number of bytes which were sent.
       *file* must be a regular file object opened in binary mode. If
-      :mod:`os.sendfile` is not available (e.g. Windows) or *file* is not a
-      regular file :meth:`send` will be used instead. *offset* tells from where to
+      :mod:`os.sendfile` or :c:func:`!TransmitFile` is not available or
+      *file* is not a regular file :meth:`send` will be used
+      instead. *offset* tells from where to
       start reading the file. If specified, *count* is the total number of bytes
       to transmit as opposed to sending the file until EOF is reached. File
       position is updated on return or also in case of error in which case
@@ -2059,6 +2061,10 @@ Socket Objects
 
       .. versionadded:: 3.5
 
+      .. versionchanged:: next
+         On Windows, :c:func:`!TransmitFile` is now used instead of falling
+         back to :meth:`send`.
+
    .. method:: set_inheritable(inheritable)
 
       Set the :ref:`inheritable flag <fd_inheritance>` of the socket's file

Doc/whatsnew/3.16.rst

@@ -243,6 +243,15 @@ shlex
   (Contributed by Jay Berry in :gh:`148846`.)
 
 
+socket
+------
+
+* :meth:`socket.socket.sendfile` now uses the :c:func:`!TransmitFile` system
+  call on Windows, instead of falling back to a slower
+  :meth:`~socket.socket.send` loop.
+  (Contributed by An Long in :gh:`65920`.)
+
+
 tkinter
 -------
 

Lib/socket.py

@@ -58,6 +58,13 @@
 from enum import IntEnum, IntFlag
 from functools import partial
 
+try:
+    import _overlapped
+    import msvcrt
+except ImportError:
+    _overlapped = None
+    msvcrt = None
+
 try:
     import errno
 except ImportError:
@@ -467,6 +474,66 @@ def _sendfile_use_send(self, file, offset=0, count=None):
             if total_sent > 0 and hasattr(file, 'seek'):
                 file.seek(offset + total_sent)
 
+    if _overlapped and msvcrt:
+        def _sendfile_use_transmitfile(self, file, offset=0, count=None):
+            self._check_sendfile_params(file, offset, count)
+            timeout = self.gettimeout()
+            if timeout == 0:
+                raise ValueError("non-blocking sockets are not supported")
+            try:
+                fileno = file.fileno()
+            except (AttributeError, io.UnsupportedOperation) as err:
+                raise _GiveupOnSendfile(err)  # not a regular file
+            try:
+                os.fstat(fileno)
+            except OSError as err:
+                raise _GiveupOnSendfile(err)  # not a regular file
+            sock_fileno = self.fileno()
+            file_handle = msvcrt.get_osfhandle(fileno)
+            timeout_ms = _overlapped.INFINITE
+            if timeout is not None:
+                timeout_ms = int(timeout * 1000)
+
+            max_count = 0xffff_ffff
+            total_sent = 0
+            remaining = count
+            try:
+                while True:
+                    chunk_offset = offset + total_sent
+                    offset_low = chunk_offset & 0xffff_ffff
+                    offset_high = (chunk_offset >> 32) & 0xffff_ffff
+                    if remaining is None:
+                        chunk_count = 0
+                    else:
+                        chunk_count = min(remaining, max_count)
+                        if chunk_count <= 0:
+                            break
+
+                    ov = _overlapped.Overlapped()
+                    ov.TransmitFile(sock_fileno, file_handle, offset_low,
+                                    offset_high, chunk_count, 0, 0)
+                    try:
+                        sent = ov.getresultex(timeout_ms, False)
+                    except WindowsError as e:
+                        if e.winerror == 258:
+                            raise TimeoutError('timed out')
+                        raise
+
+                    total_sent += sent
+
+                    if remaining is None:
+                        if sent == 0:
+                            break
+                    else:
+                        remaining -= sent
+                        if sent < chunk_count:
+                            break
+
+                return total_sent
+            finally:
+                if total_sent > 0 and hasattr(file, 'seek'):
+                    file.seek(offset + total_sent)
+
     def _check_sendfile_params(self, file, offset, count):
         if 'b' not in getattr(file, 'mode', 'b'):
             raise ValueError("file should be opened in binary mode")
@@ -484,10 +551,10 @@ def sendfile(self, file, offset=0, count=None):
         """sendfile(file[, offset[, count]]) -> sent
 
         Send a file until EOF is reached by using high-performance
-        os.sendfile() and return the total number of bytes which
-        were sent.
+        os.sendfile() or TransmitFile() and return the total number of
+        bytes which were sent.
         *file* must be a regular file object opened in binary mode.
-        If os.sendfile() is not available (e.g. Windows) or file is
+        If os.sendfile() or TransmitFile() is not available or file is
         not a regular file socket.send() will be used instead.
         *offset* tells from where to start reading the file.
         If specified, *count* is the total number of bytes to transmit
@@ -499,6 +566,11 @@ def sendfile(self, file, offset=0, count=None):
         Non-blocking sockets are not supported.
         """
         try:
+            if sys.platform == "win32":
+                sendfile_use_transmitfile = getattr(
+                    self, "_sendfile_use_transmitfile", None)
+                if sendfile_use_transmitfile is not None:
+                    return sendfile_use_transmitfile(file, offset, count)
             return self._sendfile_use_sendfile(file, offset, count)
         except _GiveupOnSendfile:
             return self._sendfile_use_send(file, offset, count)

Lib/test/test_socket.py

@@ -7101,6 +7101,15 @@ def meth_from_sock(self, sock):
         return getattr(sock, "_sendfile_use_sendfile")
 
 
+@unittest.skipUnless(sys.platform == "win32", "Windows only test.")
+class SendfileUsingTransmitfileTest(SendfileUsingSendTest):
+    """
+    Test the TransmitFile() implementation of socket.sendfile().
+    """
+    def meth_from_sock(self, sock):
+        return getattr(sock, "_sendfile_use_transmitfile")
+
+
 @unittest.skipUnless(HAVE_SOCKET_ALG, 'AF_ALG required')
 class LinuxKernelCryptoAPI(unittest.TestCase):
     # tests for AF_ALG

Misc/NEWS.d/next/Windows/2023-11-23-22-25-16.gh-issue-65920.6wWsHN.rst

@@ -0,0 +1 @@
+Use :c:func:`!TransmitFile` on Windows to implement :func:`!socket.sendfile`.

Modules/overlapped.c

@@ -877,46 +877,14 @@ _overlapped_Overlapped_cancel_impl(OverlappedObject *self)
     Py_RETURN_NONE;
 }
 
-/*[clinic input]
-_overlapped.Overlapped.getresult
-
-    wait: BOOL(c_default='FALSE') = False
-    /
-
-Retrieve result of operation.
-
-If wait is true then it blocks until the operation is finished.  If
-wait is false and the operation is still pending then an error is
-raised.
-[clinic start generated code]*/
-
 static PyObject *
-_overlapped_Overlapped_getresult_impl(OverlappedObject *self, BOOL wait)
-/*[clinic end generated code: output=8c9bd04d08994f6c input=852fbd817cbd2b3d]*/
+check_getresult_error(OverlappedObject *self, DWORD transferred)
 {
-    DWORD transferred = 0;
-    BOOL ret;
-    DWORD err;
-    PyObject *addr;
+    PyObject *addr = NULL;
     PyObject *transferred_obj;
+    DWORD err = self->error;
 
-    if (self->type == TYPE_NONE) {
-        PyErr_SetString(PyExc_ValueError, "operation not yet attempted");
-        return NULL;
-    }
-
-    if (self->type == TYPE_NOT_STARTED) {
-        PyErr_SetString(PyExc_ValueError, "operation failed to start");
-        return NULL;
-    }
-
-    Py_BEGIN_ALLOW_THREADS
-    ret = GetOverlappedResult(self->handle, &self->overlapped, &transferred,
-                              wait);
-    Py_END_ALLOW_THREADS
-
-    self->error = err = ret ? ERROR_SUCCESS : GetLastError();
-    switch (err) {
+    switch (self->error) {
         case ERROR_SUCCESS:
         case ERROR_MORE_DATA:
             break;
@@ -1002,6 +970,88 @@ _overlapped_Overlapped_getresult_impl(OverlappedObject *self, BOOL wait)
     }
 }
 
+/*[clinic input]
+_overlapped.Overlapped.getresult
+
+    wait: BOOL(c_default='FALSE') = False
+    /
+
+Retrieve result of operation.
+
+If wait is true then it blocks until the operation is finished.  If
+wait is false and the operation is still pending then an error is
+raised.
+[clinic start generated code]*/
+
+static PyObject *
+_overlapped_Overlapped_getresult_impl(OverlappedObject *self, BOOL wait)
+/*[clinic end generated code: output=8c9bd04d08994f6c input=852fbd817cbd2b3d]*/
+{
+    DWORD transferred = 0;
+    BOOL ret;
+
+    if (self->type == TYPE_NONE) {
+        PyErr_SetString(PyExc_ValueError, "operation not yet attempted");
+        return NULL;
+    }
+
+    if (self->type == TYPE_NOT_STARTED) {
+        PyErr_SetString(PyExc_ValueError, "operation failed to start");
+        return NULL;
+    }
+
+    Py_BEGIN_ALLOW_THREADS
+    ret = GetOverlappedResult(self->handle, &self->overlapped, &transferred,
+                              wait);
+    Py_END_ALLOW_THREADS
+
+    self->error = ret ? ERROR_SUCCESS : GetLastError();
+
+    return check_getresult_error(self, transferred);
+}
+
+/*[clinic input]
+_overlapped.Overlapped.getresultex
+
+    milliseconds: DWORD
+    alertable: BOOL
+    /
+
+Retrieve result of operation, waiting up to milliseconds.
+
+If the operation does not finish within milliseconds, a
+WAIT_TIMEOUT error is raised.  If alertable is true the wait can
+also be interrupted to run queued APCs.
+[clinic start generated code]*/
+
+static PyObject *
+_overlapped_Overlapped_getresultex_impl(OverlappedObject *self,
+                                        DWORD milliseconds, BOOL alertable)
+/*[clinic end generated code: output=ce0eb6ffb9618e54 input=9891d8ae4afe2b00]*/
+{
+    DWORD transferred = 0;
+    BOOL ret;
+
+    if (self->type == TYPE_NONE) {
+        PyErr_SetString(PyExc_ValueError, "operation not yet attempted");
+        return NULL;
+    }
+
+    if (self->type == TYPE_NOT_STARTED) {
+        PyErr_SetString(PyExc_ValueError, "operation failed to start");
+        return NULL;
+    }
+
+    Py_BEGIN_ALLOW_THREADS
+    ret = GetOverlappedResultEx(self->handle, &self->overlapped, &transferred,
+                                milliseconds, alertable);
+    Py_END_ALLOW_THREADS
+
+    self->error = ret ? ERROR_SUCCESS : GetLastError();
+
+    return check_getresult_error(self, transferred);
+}
+
 static PyObject *
 do_ReadFile(OverlappedObject *self, HANDLE handle,
             char *bufstart, DWORD buflen)
@@ -1952,6 +2002,7 @@ _overlapped_Overlapped_WSARecvFromInto_impl(OverlappedObject *self,
 
 static PyMethodDef Overlapped_methods[] = {
     _OVERLAPPED_OVERLAPPED_GETRESULT_METHODDEF
+    _OVERLAPPED_OVERLAPPED_GETRESULTEX_METHODDEF
     _OVERLAPPED_OVERLAPPED_CANCEL_METHODDEF
     _OVERLAPPED_OVERLAPPED_READFILE_METHODDEF
     _OVERLAPPED_OVERLAPPED_READFILEINTO_METHODDEF