From 95d1e2d384e549e202b4a463fb24c18cc4aece22 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 30 Apr 2021 09:30:44 +0100 Subject: [PATCH 1/6] Add failing test case for 'content=io.BytesIO(...)' --- tests/test_content.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_content.py b/tests/test_content.py index b105966198..2a273a133f 100644 --- a/tests/test_content.py +++ b/tests/test_content.py @@ -48,6 +48,18 @@ async def test_bytes_content(): assert async_content == b"Hello, world!" +@pytest.mark.asyncio +async def test_bytesio_content(): + headers, stream = encode_request(content=io.BytesIO(b"Hello, world!")) + assert isinstance(stream, typing.Iterable) + assert not isinstance(stream, typing.AsyncIterable) + + content = b"".join([part for part in stream]) + + assert headers == {"Content-Length": "13"} + assert content == b"Hello, world!" + + @pytest.mark.asyncio async def test_iterator_content(): def hello_world(): From 3c576c29d62e4eb0401933e7adde5f8041bcc947 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 30 Apr 2021 09:52:09 +0100 Subject: [PATCH 2/6] Refactor peek_filelike_length to return an Optional[int] --- httpx/_multipart.py | 5 ++--- httpx/_utils.py | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/httpx/_multipart.py b/httpx/_multipart.py index cb23d0cfa5..36bae664e3 100644 --- a/httpx/_multipart.py +++ b/httpx/_multipart.py @@ -93,9 +93,8 @@ def get_length(self) -> int: return len(headers) + len(to_bytes(self.file)) # Let's do our best not to read `file` into memory. - try: - file_length = peek_filelike_length(self.file) - except OSError: + file_length = peek_filelike_length(self.file) + if file_length is None: # As a last resort, read file and cache contents for later. assert not hasattr(self, "_data") self._data = to_bytes(self.file.read()) diff --git a/httpx/_utils.py b/httpx/_utils.py index dcdc5c3aa5..9255fadfe7 100644 --- a/httpx/_utils.py +++ b/httpx/_utils.py @@ -342,7 +342,7 @@ def guess_content_type(filename: typing.Optional[str]) -> typing.Optional[str]: return None -def peek_filelike_length(stream: typing.IO) -> int: +def peek_filelike_length(stream: typing.IO) -> typing.Optional[int]: """ Given a file-like stream object, return its length in number of bytes without reading it into memory. @@ -360,7 +360,7 @@ def peek_filelike_length(stream: typing.IO) -> int: stream.seek(offset) except OSError: # Not even that? Sorry, we're doomed... - raise + return None else: return length else: From 5a46c171b0a486467283ff611dde643e9d93cb90 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 30 Apr 2021 10:13:34 +0100 Subject: [PATCH 3/6] Peek filelength on file-like objects when rendering 'content=...' --- httpx/_content.py | 13 +++++++++---- httpx/_utils.py | 15 +++++++-------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/httpx/_content.py b/httpx/_content.py index 9c7c1ff225..86f3c7c254 100644 --- a/httpx/_content.py +++ b/httpx/_content.py @@ -17,7 +17,7 @@ from ._multipart import MultipartStream from ._transports.base import AsyncByteStream, SyncByteStream from ._types import RequestContent, RequestData, RequestFiles, ResponseContent -from ._utils import primitive_value_to_str +from ._utils import peek_filelike_length, primitive_value_to_str class ByteStream(AsyncByteStream, SyncByteStream): @@ -82,12 +82,17 @@ def encode_content( if isinstance(content, (bytes, str)): body = content.encode("utf-8") if isinstance(content, str) else content - content_length = str(len(body)) - headers = {"Content-Length": content_length} if body else {} + content_length = len(body) + headers = {"Content-Length": str(content_length)} if body else {} return headers, ByteStream(body) elif isinstance(content, Iterable): - headers = {"Transfer-Encoding": "chunked"} + content_length_or_none = peek_filelike_length(content) + + if content_length_or_none is None: + headers = {"Transfer-Encoding": "chunked"} + else: + headers = {"Content-Length": str(content_length_or_none)} return headers, IteratorByteStream(content) # type: ignore elif isinstance(content, AsyncIterable): diff --git a/httpx/_utils.py b/httpx/_utils.py index 9255fadfe7..30ab2ed5a6 100644 --- a/httpx/_utils.py +++ b/httpx/_utils.py @@ -342,7 +342,7 @@ def guess_content_type(filename: typing.Optional[str]) -> typing.Optional[str]: return None -def peek_filelike_length(stream: typing.IO) -> typing.Optional[int]: +def peek_filelike_length(stream: typing.Any) -> typing.Optional[int]: """ Given a file-like stream object, return its length in number of bytes without reading it into memory. @@ -350,7 +350,9 @@ def peek_filelike_length(stream: typing.IO) -> typing.Optional[int]: try: # Is it an actual file? fd = stream.fileno() - except OSError: + # Yup, seems to be an actual file. + length = os.fstat(fd).st_size + except (AttributeError, OSError): # No... Maybe it's something that supports random access, like `io.BytesIO`? try: # Assuming so, go to end of stream to figure out its length, @@ -358,14 +360,11 @@ def peek_filelike_length(stream: typing.IO) -> typing.Optional[int]: offset = stream.tell() length = stream.seek(0, os.SEEK_END) stream.seek(offset) - except OSError: + except (AttributeError, OSError): # Not even that? Sorry, we're doomed... return None - else: - return length - else: - # Yup, seems to be an actual file. - return os.fstat(fd).st_size + + return length class Timer: From 897f30cf29691b1f2f5684c93d1a2f2c1be1df1e Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Mon, 10 May 2021 16:15:45 +0100 Subject: [PATCH 4/6] Switch to USE_CLIENT_DEFAULT --- httpx/__init__.py | 3 +- httpx/_client.py | 105 ++++++++++++++++++++++++++-------------------- 2 files changed, 61 insertions(+), 47 deletions(-) diff --git a/httpx/__init__.py b/httpx/__init__.py index 9a27790f4c..ac14482b64 100644 --- a/httpx/__init__.py +++ b/httpx/__init__.py @@ -1,7 +1,7 @@ from .__version__ import __description__, __title__, __version__ from ._api import delete, get, head, options, patch, post, put, request, stream from ._auth import Auth, BasicAuth, DigestAuth -from ._client import AsyncClient, Client +from ._client import AsyncClient, Client, USE_CLIENT_DEFAULT from ._config import Limits, Proxy, Timeout, create_ssl_context from ._content import ByteStream from ._exceptions import ( @@ -111,6 +111,7 @@ "TransportError", "UnsupportedProtocol", "URL", + "USE_CLIENT_DEFAULT", "WriteError", "WriteTimeout", "WSGITransport", diff --git a/httpx/_client.py b/httpx/_client.py index 2928577b83..7d0723be0a 100644 --- a/httpx/_client.py +++ b/httpx/_client.py @@ -12,11 +12,9 @@ DEFAULT_LIMITS, DEFAULT_MAX_REDIRECTS, DEFAULT_TIMEOUT_CONFIG, - UNSET, Limits, Proxy, Timeout, - UnsetType, ) from ._decoders import SUPPORTED_DECODERS from ._exceptions import ( @@ -65,6 +63,13 @@ U = typing.TypeVar("U", bound="AsyncClient") +class UseClientDefault: + pass # pragma: nocover + + +USE_CLIENT_DEFAULT = UseClientDefault() + + logger = get_logger(__name__) USER_AGENT = f"python-httpx/{__version__}" @@ -402,9 +407,13 @@ def _build_auth(self, auth: AuthTypes) -> typing.Optional[Auth]: raise TypeError(f'Invalid "auth" argument: {auth!r}') def _build_request_auth( - self, request: Request, auth: typing.Union[AuthTypes, UnsetType] = UNSET + self, + request: Request, + auth: typing.Union[AuthTypes, UseClientDefault] = USE_CLIENT_DEFAULT, ) -> Auth: - auth = self._auth if isinstance(auth, UnsetType) else self._build_auth(auth) + auth = ( + self._auth if isinstance(auth, UseClientDefault) else self._build_auth(auth) + ) if auth is not None: return auth @@ -705,9 +714,9 @@ def request( params: QueryParamTypes = None, headers: HeaderTypes = None, cookies: CookieTypes = None, - auth: typing.Union[AuthTypes, UnsetType] = UNSET, + auth: typing.Union[AuthTypes, UseClientDefault] = USE_CLIENT_DEFAULT, allow_redirects: bool = True, - timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, + timeout: typing.Union[TimeoutTypes, UseClientDefault] = USE_CLIENT_DEFAULT, ) -> Response: """ Build and send a request. @@ -761,9 +770,9 @@ def stream( params: QueryParamTypes = None, headers: HeaderTypes = None, cookies: CookieTypes = None, - auth: typing.Union[AuthTypes, UnsetType] = UNSET, + auth: typing.Union[AuthTypes, UseClientDefault] = USE_CLIENT_DEFAULT, allow_redirects: bool = True, - timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, + timeout: typing.Union[TimeoutTypes, UseClientDefault] = USE_CLIENT_DEFAULT, ) -> typing.Iterator[Response]: """ Alternative to `httpx.request()` that streams the response body @@ -803,9 +812,9 @@ def send( request: Request, *, stream: bool = False, - auth: typing.Union[AuthTypes, UnsetType] = UNSET, + auth: typing.Union[AuthTypes, UseClientDefault] = USE_CLIENT_DEFAULT, allow_redirects: bool = True, - timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, + timeout: typing.Union[TimeoutTypes, UseClientDefault] = USE_CLIENT_DEFAULT, ) -> Response: """ Send a request. @@ -824,7 +833,9 @@ def send( raise RuntimeError("Cannot send a request, as the client has been closed.") self._state = ClientState.OPENED - timeout = self.timeout if isinstance(timeout, UnsetType) else Timeout(timeout) + timeout = ( + self.timeout if isinstance(timeout, UseClientDefault) else Timeout(timeout) + ) auth = self._build_request_auth(request, auth) @@ -963,9 +974,9 @@ def get( params: QueryParamTypes = None, headers: HeaderTypes = None, cookies: CookieTypes = None, - auth: typing.Union[AuthTypes, UnsetType] = UNSET, + auth: typing.Union[AuthTypes, UseClientDefault] = USE_CLIENT_DEFAULT, allow_redirects: bool = True, - timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, + timeout: typing.Union[TimeoutTypes, UseClientDefault] = USE_CLIENT_DEFAULT, ) -> Response: """ Send a `GET` request. @@ -990,9 +1001,9 @@ def options( params: QueryParamTypes = None, headers: HeaderTypes = None, cookies: CookieTypes = None, - auth: typing.Union[AuthTypes, UnsetType] = UNSET, + auth: typing.Union[AuthTypes, UseClientDefault] = USE_CLIENT_DEFAULT, allow_redirects: bool = True, - timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, + timeout: typing.Union[TimeoutTypes, UseClientDefault] = USE_CLIENT_DEFAULT, ) -> Response: """ Send an `OPTIONS` request. @@ -1017,9 +1028,9 @@ def head( params: QueryParamTypes = None, headers: HeaderTypes = None, cookies: CookieTypes = None, - auth: typing.Union[AuthTypes, UnsetType] = UNSET, + auth: typing.Union[AuthTypes, UseClientDefault] = USE_CLIENT_DEFAULT, allow_redirects: bool = True, - timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, + timeout: typing.Union[TimeoutTypes, UseClientDefault] = USE_CLIENT_DEFAULT, ) -> Response: """ Send a `HEAD` request. @@ -1048,9 +1059,9 @@ def post( params: QueryParamTypes = None, headers: HeaderTypes = None, cookies: CookieTypes = None, - auth: typing.Union[AuthTypes, UnsetType] = UNSET, + auth: typing.Union[AuthTypes, UseClientDefault] = USE_CLIENT_DEFAULT, allow_redirects: bool = True, - timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, + timeout: typing.Union[TimeoutTypes, UseClientDefault] = USE_CLIENT_DEFAULT, ) -> Response: """ Send a `POST` request. @@ -1083,9 +1094,9 @@ def put( params: QueryParamTypes = None, headers: HeaderTypes = None, cookies: CookieTypes = None, - auth: typing.Union[AuthTypes, UnsetType] = UNSET, + auth: typing.Union[AuthTypes, UseClientDefault] = USE_CLIENT_DEFAULT, allow_redirects: bool = True, - timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, + timeout: typing.Union[TimeoutTypes, UseClientDefault] = USE_CLIENT_DEFAULT, ) -> Response: """ Send a `PUT` request. @@ -1118,9 +1129,9 @@ def patch( params: QueryParamTypes = None, headers: HeaderTypes = None, cookies: CookieTypes = None, - auth: typing.Union[AuthTypes, UnsetType] = UNSET, + auth: typing.Union[AuthTypes, UseClientDefault] = USE_CLIENT_DEFAULT, allow_redirects: bool = True, - timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, + timeout: typing.Union[TimeoutTypes, UseClientDefault] = USE_CLIENT_DEFAULT, ) -> Response: """ Send a `PATCH` request. @@ -1149,9 +1160,9 @@ def delete( params: QueryParamTypes = None, headers: HeaderTypes = None, cookies: CookieTypes = None, - auth: typing.Union[AuthTypes, UnsetType] = UNSET, + auth: typing.Union[AuthTypes, UseClientDefault] = USE_CLIENT_DEFAULT, allow_redirects: bool = True, - timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, + timeout: typing.Union[TimeoutTypes, UseClientDefault] = USE_CLIENT_DEFAULT, ) -> Response: """ Send a `DELETE` request. @@ -1391,9 +1402,9 @@ async def request( params: QueryParamTypes = None, headers: HeaderTypes = None, cookies: CookieTypes = None, - auth: typing.Union[AuthTypes, UnsetType] = UNSET, + auth: typing.Union[AuthTypes, UseClientDefault] = USE_CLIENT_DEFAULT, allow_redirects: bool = True, - timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, + timeout: typing.Union[TimeoutTypes, UseClientDefault] = USE_CLIENT_DEFAULT, ) -> Response: """ Build and send a request. @@ -1440,9 +1451,9 @@ async def stream( params: QueryParamTypes = None, headers: HeaderTypes = None, cookies: CookieTypes = None, - auth: typing.Union[AuthTypes, UnsetType] = UNSET, + auth: typing.Union[AuthTypes, UseClientDefault] = USE_CLIENT_DEFAULT, allow_redirects: bool = True, - timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, + timeout: typing.Union[TimeoutTypes, UseClientDefault] = USE_CLIENT_DEFAULT, ) -> typing.AsyncIterator[Response]: """ Alternative to `httpx.request()` that streams the response body @@ -1482,9 +1493,9 @@ async def send( request: Request, *, stream: bool = False, - auth: typing.Union[AuthTypes, UnsetType] = UNSET, + auth: typing.Union[AuthTypes, UseClientDefault] = USE_CLIENT_DEFAULT, allow_redirects: bool = True, - timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, + timeout: typing.Union[TimeoutTypes, UseClientDefault] = USE_CLIENT_DEFAULT, ) -> Response: """ Send a request. @@ -1503,7 +1514,9 @@ async def send( raise RuntimeError("Cannot send a request, as the client has been closed.") self._state = ClientState.OPENED - timeout = self.timeout if isinstance(timeout, UnsetType) else Timeout(timeout) + timeout = ( + self.timeout if isinstance(timeout, UseClientDefault) else Timeout(timeout) + ) auth = self._build_request_auth(request, auth) @@ -1649,9 +1662,9 @@ async def get( params: QueryParamTypes = None, headers: HeaderTypes = None, cookies: CookieTypes = None, - auth: typing.Union[AuthTypes, UnsetType] = UNSET, + auth: typing.Union[AuthTypes, UseClientDefault] = USE_CLIENT_DEFAULT, allow_redirects: bool = True, - timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, + timeout: typing.Union[TimeoutTypes, UseClientDefault] = USE_CLIENT_DEFAULT, ) -> Response: """ Send a `GET` request. @@ -1676,9 +1689,9 @@ async def options( params: QueryParamTypes = None, headers: HeaderTypes = None, cookies: CookieTypes = None, - auth: typing.Union[AuthTypes, UnsetType] = UNSET, + auth: typing.Union[AuthTypes, UseClientDefault] = USE_CLIENT_DEFAULT, allow_redirects: bool = True, - timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, + timeout: typing.Union[TimeoutTypes, UseClientDefault] = USE_CLIENT_DEFAULT, ) -> Response: """ Send an `OPTIONS` request. @@ -1703,9 +1716,9 @@ async def head( params: QueryParamTypes = None, headers: HeaderTypes = None, cookies: CookieTypes = None, - auth: typing.Union[AuthTypes, UnsetType] = UNSET, + auth: typing.Union[AuthTypes, UseClientDefault] = USE_CLIENT_DEFAULT, allow_redirects: bool = True, - timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, + timeout: typing.Union[TimeoutTypes, UseClientDefault] = USE_CLIENT_DEFAULT, ) -> Response: """ Send a `HEAD` request. @@ -1734,9 +1747,9 @@ async def post( params: QueryParamTypes = None, headers: HeaderTypes = None, cookies: CookieTypes = None, - auth: typing.Union[AuthTypes, UnsetType] = UNSET, + auth: typing.Union[AuthTypes, UseClientDefault] = USE_CLIENT_DEFAULT, allow_redirects: bool = True, - timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, + timeout: typing.Union[TimeoutTypes, UseClientDefault] = USE_CLIENT_DEFAULT, ) -> Response: """ Send a `POST` request. @@ -1769,9 +1782,9 @@ async def put( params: QueryParamTypes = None, headers: HeaderTypes = None, cookies: CookieTypes = None, - auth: typing.Union[AuthTypes, UnsetType] = UNSET, + auth: typing.Union[AuthTypes, UseClientDefault] = USE_CLIENT_DEFAULT, allow_redirects: bool = True, - timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, + timeout: typing.Union[TimeoutTypes, UseClientDefault] = USE_CLIENT_DEFAULT, ) -> Response: """ Send a `PUT` request. @@ -1804,9 +1817,9 @@ async def patch( params: QueryParamTypes = None, headers: HeaderTypes = None, cookies: CookieTypes = None, - auth: typing.Union[AuthTypes, UnsetType] = UNSET, + auth: typing.Union[AuthTypes, UseClientDefault] = USE_CLIENT_DEFAULT, allow_redirects: bool = True, - timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, + timeout: typing.Union[TimeoutTypes, UseClientDefault] = USE_CLIENT_DEFAULT, ) -> Response: """ Send a `PATCH` request. @@ -1835,9 +1848,9 @@ async def delete( params: QueryParamTypes = None, headers: HeaderTypes = None, cookies: CookieTypes = None, - auth: typing.Union[AuthTypes, UnsetType] = UNSET, + auth: typing.Union[AuthTypes, UseClientDefault] = USE_CLIENT_DEFAULT, allow_redirects: bool = True, - timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, + timeout: typing.Union[TimeoutTypes, UseClientDefault] = USE_CLIENT_DEFAULT, ) -> Response: """ Send a `DELETE` request. From 70bb8594631c9f0d5685ddd0fc4ef40d8d2e76c1 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Mon, 10 May 2021 16:15:51 +0100 Subject: [PATCH 5/6] Switch to USE_CLIENT_DEFAULT --- httpx/_client.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/httpx/_client.py b/httpx/_client.py index 7d0723be0a..5a1c476396 100644 --- a/httpx/_client.py +++ b/httpx/_client.py @@ -64,6 +64,23 @@ class UseClientDefault: + """ + For some parameters such as `auth=...` and `timeout=...` we need to be able + to indicate the default "unset" state, in a way that is distinctly different + to using `None`. + + The default "unset" state indicates that whatever default is set on the + client should be used. This is different to setting `None`, which + explicitly disables the parameter, possibly overriding a client default. + + For example we use `timeout=USE_CLIENT_DEFAULT` in the `request()` signature. + Omitting the `timeout` parameter will send a request using whatever default + timeout has been configured on the client. Including `timeout=None` will + ensure no timeout is used. + + Note that user code shouldn't need to use the `USE_CLIENT_DEFAULT` constant, + but it is used internally when a parameter is not included. + """ pass # pragma: nocover From 56d7ebcc5395c8999ff4a0931aed70708f0368d6 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Tue, 11 May 2021 09:06:26 +0100 Subject: [PATCH 6/6] Linting --- httpx/__init__.py | 2 +- httpx/_client.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/httpx/__init__.py b/httpx/__init__.py index ac14482b64..4af3904fd3 100644 --- a/httpx/__init__.py +++ b/httpx/__init__.py @@ -1,7 +1,7 @@ from .__version__ import __description__, __title__, __version__ from ._api import delete, get, head, options, patch, post, put, request, stream from ._auth import Auth, BasicAuth, DigestAuth -from ._client import AsyncClient, Client, USE_CLIENT_DEFAULT +from ._client import USE_CLIENT_DEFAULT, AsyncClient, Client from ._config import Limits, Proxy, Timeout, create_ssl_context from ._content import ByteStream from ._exceptions import ( diff --git a/httpx/_client.py b/httpx/_client.py index 5a1c476396..7fcc85b008 100644 --- a/httpx/_client.py +++ b/httpx/_client.py @@ -81,6 +81,7 @@ class UseClientDefault: Note that user code shouldn't need to use the `USE_CLIENT_DEFAULT` constant, but it is used internally when a parameter is not included. """ + pass # pragma: nocover