From c7707952422fc331c6b9cd493fab3049fe6ed04a Mon Sep 17 00:00:00 2001 From: Stephen Brown II Date: Sat, 27 Jul 2019 17:27:50 -0500 Subject: [PATCH 1/6] Update IDNA encoding to 2008 spec --- httpx/models.py | 7 ++++++- tests/models/test_url.py | 23 +++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/httpx/models.py b/httpx/models.py index 2f29be5f9f..74cd6fdd9a 100644 --- a/httpx/models.py +++ b/httpx/models.py @@ -8,6 +8,7 @@ from urllib.parse import parse_qsl, urlencode import chardet +import idna import hstspreload import rfc3986 @@ -94,7 +95,11 @@ def __init__( # Handle IDNA domain names. if self.components.authority: - idna_authority = self.components.authority.encode("idna").decode("ascii") + # idna.encode raises InvalidCodepoint when encountering a colon, so split + # host and port in case the latter is specified and rejoin after encoding + host_port = self.components.authority.split(":") + host_port[0] = idna.encode(host_port[0]).decode("ascii") + idna_authority = ":".join(host_port) if idna_authority != self.components.authority: self.components = self.components.copy_with(authority=idna_authority) diff --git a/tests/models/test_url.py b/tests/models/test_url.py index 7c865f5a66..a8fbd3f5ff 100644 --- a/tests/models/test_url.py +++ b/tests/models/test_url.py @@ -8,6 +8,29 @@ def test_idna_url(): url = URL("http://中国.icom.museum:80/") assert url == URL("http://xn--fiqs8s.icom.museum:80/") assert url.host == "xn--fiqs8s.icom.museum" + assert url.port == 80 + + url = URL("https://faß.de") + assert url == URL("https://xn--fa-hia.de") # IDNA 2008 + assert url.host == "xn--fa-hia.de" + assert url.port == 443 + + url = URL("https://βόλος.com:443") + assert url == URL("https://xn--nxasmm1c.com:443") # IDNA 2008 + assert url.host == "xn--nxasmm1c.com" + assert url.port == 443 + + url = URL("http://ශ්‍රී.com:444") + assert url == URL("http://xn--10cl1a0b660p.com:444") # IDNA 2008 + assert url.host == "xn--10cl1a0b660p.com" + assert url.scheme == "http" + assert url.port == 444 + + url = URL("https://نامه‌ای.com:4433") + assert url == URL("https://xn--mgba3gch31f060k.com:4433") # IDNA 2008 + assert url.host == "xn--mgba3gch31f060k.com" + assert url.scheme == "https" + assert url.port == 4433 def test_url(): From 7ddd40c54dd4e6a736459d00dead86edd0d122e2 Mon Sep 17 00:00:00 2001 From: Stephen Brown II Date: Sat, 27 Jul 2019 18:00:40 -0500 Subject: [PATCH 2/6] Add Unicode IDNA Compatibility Processing --- httpx/models.py | 2 +- tests/models/test_url.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/httpx/models.py b/httpx/models.py index 74cd6fdd9a..c3d79beb3c 100644 --- a/httpx/models.py +++ b/httpx/models.py @@ -98,7 +98,7 @@ def __init__( # idna.encode raises InvalidCodepoint when encountering a colon, so split # host and port in case the latter is specified and rejoin after encoding host_port = self.components.authority.split(":") - host_port[0] = idna.encode(host_port[0]).decode("ascii") + host_port[0] = idna.encode(host_port[0], uts46=True).decode("ascii") idna_authority = ":".join(host_port) if idna_authority != self.components.authority: self.components = self.components.copy_with(authority=idna_authority) diff --git a/tests/models/test_url.py b/tests/models/test_url.py index a8fbd3f5ff..9f627bd983 100644 --- a/tests/models/test_url.py +++ b/tests/models/test_url.py @@ -10,6 +10,11 @@ def test_idna_url(): assert url.host == "xn--fiqs8s.icom.museum" assert url.port == 80 + url = URL("http://Königsgäßchen.de") + assert url == URL("http://xn--knigsgchen-b4a3dun.de") # IDNA 2008 + assert url.host == "xn--knigsgchen-b4a3dun.de" + assert url.port == 80 + url = URL("https://faß.de") assert url == URL("https://xn--fa-hia.de") # IDNA 2008 assert url.host == "xn--fa-hia.de" From c76c0a0826d5bf776fec8f1863a0c839c31675cd Mon Sep 17 00:00:00 2001 From: Stephen Brown II Date: Sun, 28 Jul 2019 14:54:58 -0500 Subject: [PATCH 3/6] Parametrize idna test --- tests/models/test_url.py | 87 +++++++++++++++++++++++++--------------- 1 file changed, 55 insertions(+), 32 deletions(-) diff --git a/tests/models/test_url.py b/tests/models/test_url.py index 9f627bd983..fb5d5c6e45 100644 --- a/tests/models/test_url.py +++ b/tests/models/test_url.py @@ -4,38 +4,61 @@ from httpx.exceptions import InvalidURL -def test_idna_url(): - url = URL("http://中国.icom.museum:80/") - assert url == URL("http://xn--fiqs8s.icom.museum:80/") - assert url.host == "xn--fiqs8s.icom.museum" - assert url.port == 80 - - url = URL("http://Königsgäßchen.de") - assert url == URL("http://xn--knigsgchen-b4a3dun.de") # IDNA 2008 - assert url.host == "xn--knigsgchen-b4a3dun.de" - assert url.port == 80 - - url = URL("https://faß.de") - assert url == URL("https://xn--fa-hia.de") # IDNA 2008 - assert url.host == "xn--fa-hia.de" - assert url.port == 443 - - url = URL("https://βόλος.com:443") - assert url == URL("https://xn--nxasmm1c.com:443") # IDNA 2008 - assert url.host == "xn--nxasmm1c.com" - assert url.port == 443 - - url = URL("http://ශ්‍රී.com:444") - assert url == URL("http://xn--10cl1a0b660p.com:444") # IDNA 2008 - assert url.host == "xn--10cl1a0b660p.com" - assert url.scheme == "http" - assert url.port == 444 - - url = URL("https://نامه‌ای.com:4433") - assert url == URL("https://xn--mgba3gch31f060k.com:4433") # IDNA 2008 - assert url.host == "xn--mgba3gch31f060k.com" - assert url.scheme == "https" - assert url.port == 4433 +@pytest.mark.parametrize( + "given,idna,host,scheme,port", + [ + ( + "http://中国.icom.museum:80/", + "http://xn--fiqs8s.icom.museum:80/", + "xn--fiqs8s.icom.museum", + "http", + 80, + ), + ( + "http://Königsgäßchen.de", + "http://xn--knigsgchen-b4a3dun.de", + "xn--knigsgchen-b4a3dun.de", + "http", + 80, + ), + ("https://faß.de", "https://xn--fa-hia.de", "xn--fa-hia.de", "https", 443), + ( + "https://βόλος.com:443", + "https://xn--nxasmm1c.com:443", + "xn--nxasmm1c.com", + "https", + 443, + ), + ( + "http://ශ්‍රී.com:444", + "http://xn--10cl1a0b660p.com:444", + "xn--10cl1a0b660p.com", + "http", + 444, + ), + ( + "https://نامه‌ای.com:4433", + "https://xn--mgba3gch31f060k.com:4433", + "xn--mgba3gch31f060k.com", + "https", + 4433, + ), + ], + ids=[ + "http_with_port", + "unicode_tr46_compat", + "https_without_port", + "https_with_port", + "http_with_custom_port", + "https_with_custom_port", + ], +) +def test_idna_url(given, idna, host, scheme, port): + url = URL(given) + assert url == URL(idna) + assert url.host == host + assert url.scheme == scheme + assert url.port == port def test_url(): From 6e8feae27a8fdfb620bf00e845cd78eb153a574f Mon Sep 17 00:00:00 2001 From: Stephen Brown II Date: Sun, 28 Jul 2019 15:16:29 -0500 Subject: [PATCH 4/6] Use rfc3986 iri_reference for IDNA names --- httpx/models.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/httpx/models.py b/httpx/models.py index c3d79beb3c..c8634a5807 100644 --- a/httpx/models.py +++ b/httpx/models.py @@ -8,7 +8,6 @@ from urllib.parse import parse_qsl, urlencode import chardet -import idna import hstspreload import rfc3986 @@ -88,21 +87,14 @@ def __init__( ) -> None: if isinstance(url, rfc3986.uri.URIReference): self.components = url + elif isinstance(url, rfc3986.iri.IRIReference): + self.components = url.encode() elif isinstance(url, str): - self.components = rfc3986.api.uri_reference(url) + # Handle IDNA domain names. + self.components = rfc3986.api.iri_reference(url).encode() else: self.components = url.components - # Handle IDNA domain names. - if self.components.authority: - # idna.encode raises InvalidCodepoint when encountering a colon, so split - # host and port in case the latter is specified and rejoin after encoding - host_port = self.components.authority.split(":") - host_port[0] = idna.encode(host_port[0], uts46=True).decode("ascii") - idna_authority = ":".join(host_port) - if idna_authority != self.components.authority: - self.components = self.components.copy_with(authority=idna_authority) - # Normalize scheme and domain name. if self.is_absolute_url: self.components = self.components.normalize() From 3823c012df0bd91180f80398c548f86ed42fc4d3 Mon Sep 17 00:00:00 2001 From: Stephen Brown II Date: Sun, 28 Jul 2019 15:31:20 -0500 Subject: [PATCH 5/6] Add test for IRI object --- tests/models/test_url.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/models/test_url.py b/tests/models/test_url.py index fb5d5c6e45..f20ef87cb3 100644 --- a/tests/models/test_url.py +++ b/tests/models/test_url.py @@ -1,4 +1,5 @@ import pytest +import rfc3986 from httpx import URL from httpx.exceptions import InvalidURL @@ -78,6 +79,17 @@ def test_url(): assert new == URL("http://example.org:123/path/to/somewhere?abc=123#anchor") assert new.scheme == "http" +def test_iri(): + iri = rfc3986.iri.IRIReference.from_string("https://example.org:123/path/to/somewhere?abc=123#anchor") + url = URL(iri) + assert url.scheme == "https" + assert url.host == "example.org" + assert url.port == 123 + assert url.authority == "example.org:123" + assert url.path == "/path/to/somewhere" + assert url.query == "abc=123" + assert url.fragment == "anchor" + def test_url_eq_str(): url = URL("https://example.org:123/path/to/somewhere?abc=123#anchor") From 9bd13f121172e8aeac00a4939c71e6f2fb1a1a62 Mon Sep 17 00:00:00 2001 From: Seth Michael Larson Date: Tue, 30 Jul 2019 19:22:10 -0500 Subject: [PATCH 6/6] Remove test_iri as this interface has been removed --- tests/models/test_url.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/tests/models/test_url.py b/tests/models/test_url.py index f20ef87cb3..a556ed89a1 100644 --- a/tests/models/test_url.py +++ b/tests/models/test_url.py @@ -79,17 +79,6 @@ def test_url(): assert new == URL("http://example.org:123/path/to/somewhere?abc=123#anchor") assert new.scheme == "http" -def test_iri(): - iri = rfc3986.iri.IRIReference.from_string("https://example.org:123/path/to/somewhere?abc=123#anchor") - url = URL(iri) - assert url.scheme == "https" - assert url.host == "example.org" - assert url.port == 123 - assert url.authority == "example.org:123" - assert url.path == "/path/to/somewhere" - assert url.query == "abc=123" - assert url.fragment == "anchor" - def test_url_eq_str(): url = URL("https://example.org:123/path/to/somewhere?abc=123#anchor")