gh-43521: Add an opt-in gzip Content-Encoding handler to urllib.request

Add urllib.request.HTTPGzipHandler, an opt-in handler that requests
gzip-compressed responses and transparently decodes them.

The handler is not one of the default build_opener() handlers, so
urlopen()'s behaviour is unchanged unless it is added explicitly.  When
the request has no Accept-Encoding header, the handler adds
"Accept-Encoding: gzip"; if the response then carries
"Content-Encoding: gzip", the body is decoded incrementally as it is
read and the Content-Encoding and Content-Length headers, which describe
the compressed body, are removed.  A request that already carries an
Accept-Encoding header is left untouched, so a caller that means to
handle the compressed response itself still receives it.

Only gzip is handled, and the zlib module is required.

Author: soreavis

Doc/library/urllib.request.rst

@@ -512,6 +512,31 @@ The following classes are provided:
    Process HTTP error responses.
 
 
+.. class:: HTTPGzipHandler()
+
+   Request and transparently decode gzip-compressed responses.
+
+   This handler is not one of the handlers installed by :func:`build_opener`
+   by default; pass it explicitly to opt in::
+
+      opener = urllib.request.build_opener(urllib.request.HTTPGzipHandler)
+      with opener.open("http://www.example.com/") as response:
+          body = response.read()
+
+   Unless the request already has an :mailheader:`Accept-Encoding` header,
+   the handler adds ``Accept-Encoding: gzip``.  If the response is then sent
+   with ``Content-Encoding: gzip``, its body is decoded as it is read and the
+   :mailheader:`Content-Encoding` and :mailheader:`Content-Length` headers,
+   which describe the compressed body, are removed.  If the request already
+   carries an :mailheader:`Accept-Encoding` header, the response is returned
+   unchanged, so a caller that asked for compressed data can decode it itself.
+
+   Only ``gzip`` is supported.  The :mod:`zlib` module is required;
+   constructing the handler without it raises :exc:`ImportError`.
+
+   .. versionadded:: next
+
+
 .. _request-objects:
 
 Request Objects

Doc/whatsnew/3.16.rst

@@ -539,6 +539,16 @@ tkinter
   with no unit suffix to :class:`int` or :class:`float`.
   (Contributed by Serhiy Storchaka in :gh:`153513`.)
 
+urllib.request
+--------------
+
+* Add :class:`~urllib.request.HTTPGzipHandler`, an opt-in handler that
+  requests gzip-compressed responses (by adding an ``Accept-Encoding: gzip``
+  header) and transparently decodes them.  It is not installed by
+  :func:`~urllib.request.build_opener` by default; add it explicitly to opt
+  in.
+  (Contributed by Julian Soreavis in :gh:`43521`.)
+
 xml
 ---
 

Lib/test/test_urllib2.py

@@ -1893,6 +1893,149 @@ def test_invalid_closed(self):
         self.assertTrue(conn.fakesock.closed, "Connection not closed")
 
 
+class FakeGzipResponse:
+    # Minimal stand-in for the http.client.HTTPResponse that a response
+    # processor sees: headers plus a readable, closable body.
+    def __init__(self, data, headers, code=200):
+        self._file = io.BytesIO(data)
+        self.headers = headers
+        self.code = code
+        self.msg = "OK"
+        self.url = "http://example.com/"
+        self.closed = False
+
+    def read(self, amt=-1):
+        return self._file.read(amt)
+
+    def close(self):
+        self.closed = True
+        self._file.close()
+
+
+class HTTPGzipHandlerTests(unittest.TestCase):
+
+    body = b"The quick brown fox jumps over the lazy dog.\n" * 200
+
+    def make_response(self, data, content_encoding="gzip", content_length=True):
+        import email.message
+        headers = email.message.Message()
+        if content_encoding is not None:
+            headers["Content-Encoding"] = content_encoding
+        if content_length:
+            headers["Content-Length"] = str(len(data))
+        headers["Content-Type"] = "text/plain"
+        return FakeGzipResponse(data, headers)
+
+    def gzip_bytes(self, data):
+        import gzip
+        return gzip.compress(data)
+
+    def opened_request(self, **kwargs):
+        h = urllib.request.HTTPGzipHandler()
+        req = Request("http://example.com/", **kwargs)
+        h.http_request(req)
+        return h, req
+
+    def test_not_installed_by_default(self):
+        # Opt-in: build_opener() does not install it, but accepts it explicitly.
+        o = urllib.request.build_opener()
+        self.assertFalse(any(isinstance(h, urllib.request.HTTPGzipHandler)
+                             for h in o.handlers))
+        o = urllib.request.build_opener(urllib.request.HTTPGzipHandler)
+        self.assertTrue(any(isinstance(h, urllib.request.HTTPGzipHandler)
+                            for h in o.handlers))
+
+    def test_request_adds_accept_encoding(self):
+        h, req = self.opened_request()
+        self.assertEqual(req.get_header("Accept-encoding"), "gzip")
+
+    def test_request_keeps_user_accept_encoding(self):
+        h, req = self.opened_request(headers={"Accept-Encoding": "identity"})
+        self.assertEqual(req.get_header("Accept-encoding"), "identity")
+
+    def test_response_is_decoded(self):
+        h, req = self.opened_request()
+        response = self.make_response(self.gzip_bytes(self.body))
+        with h.http_response(req, response) as result:
+            self.assertEqual(result.read(), self.body)
+
+    def test_response_headers_stripped(self):
+        h, req = self.opened_request()
+        response = self.make_response(self.gzip_bytes(self.body))
+        result = h.http_response(req, response)
+        info = result.info()
+        self.assertIsNone(info.get("Content-Encoding"))
+        self.assertIsNone(info.get("Content-Length"))
+        self.assertEqual(info.get("Content-Type"), "text/plain")
+        self.assertEqual(result.status, 200)
+        result.close()
+
+    def test_response_streamed_in_chunks(self):
+        h, req = self.opened_request()
+        response = self.make_response(self.gzip_bytes(self.body))
+        result = h.http_response(req, response)
+        chunks = []
+        while chunk := result.read(64):
+            self.assertLessEqual(len(chunk), 64)
+            chunks.append(chunk)
+        self.assertEqual(b"".join(chunks), self.body)
+        result.close()
+
+    def test_multiple_gzip_members_decoded(self):
+        # A response may be several concatenated gzip members; all of them
+        # must be decoded, not just the first.
+        h, req = self.opened_request()
+        comp = self.gzip_bytes(b"AAA") + self.gzip_bytes(b"BBB")
+        response = self.make_response(comp)
+        with h.http_response(req, response) as result:
+            self.assertEqual(result.read(), b"AAABBB")
+
+    def test_trailing_junk_after_member_ignored(self):
+        # Bytes after the final member that are not another gzip member are
+        # tolerated: the body decodes without error and is not corrupted.
+        h, req = self.opened_request()
+        comp = self.gzip_bytes(self.body) + b"trailing garbage"
+        response = self.make_response(comp)
+        with h.http_response(req, response) as result:
+            self.assertEqual(result.read(), self.body)
+
+    def test_user_accept_encoding_passes_through(self):
+        # A caller-supplied Accept-Encoding is left to the caller to decode:
+        # the gzipped response is returned untouched.
+        h, req = self.opened_request(headers={"Accept-Encoding": "gzip"})
+        response = self.make_response(self.gzip_bytes(self.body))
+        result = h.http_response(req, response)
+        self.assertIs(result, response)
+        self.assertEqual(result.headers.get("Content-Encoding"), "gzip")
+
+    def test_non_gzip_response_passes_through(self):
+        h, req = self.opened_request()
+        response = self.make_response(self.body, content_encoding=None)
+        result = h.http_response(req, response)
+        self.assertIs(result, response)
+
+    def test_truncated_stream_raises(self):
+        h, req = self.opened_request()
+        comp = self.gzip_bytes(self.body)
+        response = self.make_response(comp[:len(comp) // 2])
+        result = h.http_response(req, response)
+        with self.assertRaises(EOFError):
+            result.read()
+        result.close()
+
+    def test_close_propagates(self):
+        h, req = self.opened_request()
+        response = self.make_response(self.gzip_bytes(self.body))
+        result = h.http_response(req, response)
+        result.close()
+        self.assertTrue(response.closed)
+
+    def test_requires_zlib(self):
+        with mock.patch.object(urllib.request, "zlib", None):
+            with self.assertRaises(ImportError):
+                urllib.request.HTTPGzipHandler()
+
+
 class MiscTests(unittest.TestCase):
 
     def opener_has_handler(self, opener, handler_class):

Lib/urllib/request.py

@@ -84,6 +84,7 @@
 import base64
 import bisect
 import contextlib
+import copy
 import email
 import hashlib
 import http.client
@@ -113,6 +114,11 @@
 else:
     _have_ssl = True
 
+try:
+    import zlib
+except ImportError:
+    zlib = None
+
 __all__ = [
     # Classes
     'Request', 'OpenerDirector', 'BaseHandler', 'HTTPDefaultErrorHandler',
@@ -122,7 +128,7 @@
     'HTTPBasicAuthHandler', 'ProxyBasicAuthHandler', 'AbstractDigestAuthHandler',
     'HTTPDigestAuthHandler', 'ProxyDigestAuthHandler', 'HTTPHandler',
     'FileHandler', 'FTPHandler', 'CacheFTPHandler', 'DataHandler',
-    'UnknownHandler', 'HTTPErrorProcessor',
+    'UnknownHandler', 'HTTPErrorProcessor', 'HTTPGzipHandler',
     # Functions
     'urlopen', 'install_opener', 'build_opener',
     'pathname2url', 'url2pathname', 'getproxies',
@@ -1392,6 +1398,107 @@ def http_response(self, request, response):
     https_request = http_request
     https_response = http_response
 
+
+class _GzipReader(io.BufferedIOBase):
+    """Incrementally decode a gzip Content-Encoding response body."""
+
+    def __init__(self, fp):
+        self._fp = fp
+        self._decompressor = zlib.decompressobj(16 + zlib.MAX_WBITS)
+        self._buffer = b''
+        self._leftover = b''
+        self._got_data = False
+        self._eof = False
+
+    def readable(self):
+        return True
+
+    def _refill(self):
+        while True:
+            raw = self._leftover or self._fp.read(io.DEFAULT_BUFFER_SIZE)
+            self._leftover = b''
+            if not raw:
+                self._eof = True
+                data = self._decompressor.flush()
+                if self._got_data and not self._decompressor.eof:
+                    raise EOFError('compressed response ended before the '
+                                   'end-of-stream marker was reached')
+                return data
+            self._got_data = True
+            if self._decompressor.eof:
+                # The previous member ended; the remaining bytes start the
+                # next concatenated member.  Decode it with a fresh
+                # decompressor, and tolerate trailing bytes that are not
+                # another member, as the reader did before.
+                self._decompressor = zlib.decompressobj(16 + zlib.MAX_WBITS)
+                try:
+                    data = self._decompressor.decompress(raw)
+                except zlib.error:
+                    self._eof = True
+                    return b''
+            else:
+                data = self._decompressor.decompress(raw)
+            self._leftover = self._decompressor.unused_data
+            if data:
+                return data
+
+    def read(self, size=-1):
+        if size is None:
+            size = -1
+        while not self._eof and (size < 0 or len(self._buffer) < size):
+            self._buffer += self._refill()
+        if size < 0:
+            data, self._buffer = self._buffer, b''
+        else:
+            data, self._buffer = self._buffer[:size], self._buffer[size:]
+        return data
+
+    def close(self):
+        try:
+            fp = self._fp
+            if fp is not None:
+                self._fp = None
+                fp.close()
+        finally:
+            super().close()
+
+
+class HTTPGzipHandler(BaseHandler):
+    """Request and transparently decode gzip Content-Encoding responses.
+
+    This handler is not installed by build_opener() by default; add it
+    explicitly to opt in.  It sends Accept-Encoding: gzip unless the request
+    already carries an Accept-Encoding header, and decodes the response only
+    when it added that header, so a caller supplying its own Accept-Encoding
+    still receives the raw response.
+    """
+
+    def __init__(self):
+        if zlib is None:
+            raise ImportError('the zlib module is required for gzip decoding')
+
+    def http_request(self, request):
+        if not request.has_header('Accept-encoding'):
+            request.add_unredirected_header('Accept-encoding', 'gzip')
+            request._gzip_injected = True
+        return request
+
+    def http_response(self, request, response):
+        if (getattr(request, '_gzip_injected', False) and
+                response.headers.get('Content-Encoding', '').lower() == 'gzip'):
+            headers = copy.copy(response.headers)
+            del headers['Content-Encoding']
+            del headers['Content-Length']
+            result = addinfourl(_GzipReader(response), headers,
+                                response.url, response.code)
+            result.msg = response.msg
+            return result
+        return response
+
+    https_request = http_request
+    https_response = http_response
+
+
 class UnknownHandler(BaseHandler):
     def unknown_open(self, req):
         type = req.type

Misc/NEWS.d/next/Library/2026-07-19-12-00-00.gh-issue-43521.aYO2xG.rst

@@ -0,0 +1,5 @@
+Add :class:`urllib.request.HTTPGzipHandler`, an opt-in handler that sends an
+``Accept-Encoding: gzip`` request header and transparently decodes
+gzip-compressed responses.  It is not one of the default
+:func:`~urllib.request.build_opener` handlers, so it must be added
+explicitly.