From 3f72491730c3c8b05ff97367cf9ba1223398dccc Mon Sep 17 00:00:00 2001 From: Stein Karlsen Date: Thu, 25 Jan 2018 19:13:12 +0100 Subject: [PATCH 1/8] bpo-32498: Handle bytes as input to urllib.parse.unquote --- Lib/test/test_urllib.py | 20 ++++++++++++++++++-- Lib/urllib/parse.py | 2 ++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py index c292d74f84a93c1..69c8ce865ec26be 100644 --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@ -968,8 +968,6 @@ def test_unquoting(self): "%s" % result) self.assertRaises((TypeError, AttributeError), urllib.parse.unquote, None) self.assertRaises((TypeError, AttributeError), urllib.parse.unquote, ()) - with support.check_warnings(('', BytesWarning), quiet=True): - self.assertRaises((TypeError, AttributeError), urllib.parse.unquote, b'') def test_unquoting_badpercent(self): # Test unquoting on bad percent-escapes @@ -1129,6 +1127,24 @@ def test_unquote_with_unicode(self): self.assertEqual(expect, result, "using unquote(): %r != %r" % (expect, result)) + def test_unquoting_with_bytes_input(self): + # Make sure unquoting of all ASCII values works + escape_list = [] + for num in range(128): + given = bytes([num]) + expect = chr(num) + result = urllib.parse.unquote(given) + self.assertEqual(expect, result, + "using unquote(): %r != %r" % (expect, result)) + escape_list.append(given) + escape_string = b''.join(escape_list) + del escape_list + result = urllib.parse.unquote(escape_string) + self.assertEqual(result.count('%'), 1, + "using unquote(): not all characters escaped: " + "%s" % result) + + class urlencode_Tests(unittest.TestCase): """Tests for urlencode()""" diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index f21b8eb49cca229..a781aedde79935a 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -611,6 +611,8 @@ def unquote(string, encoding='utf-8', errors='replace'): unquote('abc%20def') -> 'abc def'. """ + if isinstance(string, bytes): + return unquote_to_bytes(string).decode(encoding, errors) if '%' not in string: string.split return string From 6f312b046ca5fd30f4cfe9de63f50ba0956a371e Mon Sep 17 00:00:00 2001 From: Stein Karlsen Date: Sun, 17 Jun 2018 21:03:27 +0200 Subject: [PATCH 2/8] Added news item --- .../next/Library/2018-06-17-21-02-25.bpo-32498.La3TZz.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2018-06-17-21-02-25.bpo-32498.La3TZz.rst diff --git a/Misc/NEWS.d/next/Library/2018-06-17-21-02-25.bpo-32498.La3TZz.rst b/Misc/NEWS.d/next/Library/2018-06-17-21-02-25.bpo-32498.La3TZz.rst new file mode 100644 index 000000000000000..ad2f9b7068723e6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-06-17-21-02-25.bpo-32498.La3TZz.rst @@ -0,0 +1,2 @@ +Made ``urllib.parse.unquote()`` polymorphic, accepting both string and +bytes. Patch by Stein Karlsen. From 94748b98ddb32c10220564c05ee0c39e241e2ed9 Mon Sep 17 00:00:00 2001 From: Stein Karlsen Date: Sat, 13 Jul 2019 12:05:06 +0200 Subject: [PATCH 3/8] bpo-32498: Handle bytes as input to urllib.parse.unquote Test non-ascii input as bytes. --- Lib/test/test_urllib.py | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py index aa7dd9120df2730..8ce0b2f06548921 100644 --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@ -1200,21 +1200,17 @@ def test_unquote_with_unicode(self): "using unquote(): %r != %r" % (expect, result)) def test_unquoting_with_bytes_input(self): - # Make sure unquoting of all ASCII values works - escape_list = [] - for num in range(128): - given = bytes([num]) - expect = chr(num) - result = urllib.parse.unquote(given) - self.assertEqual(expect, result, - "using unquote(): %r != %r" % (expect, result)) - escape_list.append(given) - escape_string = b''.join(escape_list) - del escape_list - result = urllib.parse.unquote(escape_string) - self.assertEqual(result.count('%'), 1, - "using unquote(): not all characters escaped: " - "%s" % result) + given = b'bl\xc3\xa5b\xc3\xa6rsyltet\xc3\xb8y' + expect = 'bl\u00e5b\u00e6rsyltet\u00f8y' + result = urllib.parse.unquote(given) + self.assertEqual(expect, result, + "using unquote(): %r != %r" % (expect, result)) + + given = b'bl%c3%a5b%c3%a6rsyltet%c3%b8j' + expect = 'bl\u00e5b\u00e6rsyltet\u00f8j' + result = urllib.parse.unquote(given) + self.assertEqual(expect, result, + "using unquote(): %r != %r" % (expect, result)) class urlencode_Tests(unittest.TestCase): From c868099bab1a281a5f4e4ef05f87a9bc2af991c0 Mon Sep 17 00:00:00 2001 From: Stein Karlsen Date: Sun, 28 Jul 2019 11:46:58 +0200 Subject: [PATCH 4/8] bpo-32498: Handle bytes as input to urllib.parse.unquote Update documentation of changed behaviour in version 3.9. --- Doc/library/urllib.parse.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst index 49276daa7ff43f2..53eeedba91770a1 100644 --- a/Doc/library/urllib.parse.rst +++ b/Doc/library/urllib.parse.rst @@ -571,7 +571,7 @@ task isn't already covered by the URL parsing functions above. percent-encoded sequences into Unicode characters, as accepted by the :meth:`bytes.decode` method. - *string* must be a :class:`str`. + *string* may be either a :class:`str` or a :class:`bytes`. *encoding* defaults to ``'utf-8'``. *errors* defaults to ``'replace'``, meaning invalid sequences are replaced @@ -579,6 +579,11 @@ task isn't already covered by the URL parsing functions above. Example: ``unquote('/El%20Ni%C3%B1o/')`` yields ``'/El Niño/'``. + .. versionchanged:: 3.9 + String parameter supports bytes and string objects. + + + .. function:: unquote_plus(string, encoding='utf-8', errors='replace') From e7dfe55d55fb8ae1663d25bf6918b179e74d47c1 Mon Sep 17 00:00:00 2001 From: Stein Karlsen Date: Sun, 13 Oct 2019 13:21:53 +0200 Subject: [PATCH 5/8] Update Doc/library/urllib.parse.rst Co-Authored-By: Tal Einat --- Doc/library/urllib.parse.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst index 53eeedba91770a1..84d289bc4415c8c 100644 --- a/Doc/library/urllib.parse.rst +++ b/Doc/library/urllib.parse.rst @@ -580,7 +580,7 @@ task isn't already covered by the URL parsing functions above. Example: ``unquote('/El%20Ni%C3%B1o/')`` yields ``'/El Niño/'``. .. versionchanged:: 3.9 - String parameter supports bytes and string objects. + *string* parameter supports bytes and str objects (previously only str). From 943626ca11596479f85d7d6f6102d7a600a067d3 Mon Sep 17 00:00:00 2001 From: Stein Karlsen Date: Sun, 13 Oct 2019 19:47:57 +0200 Subject: [PATCH 6/8] bpo-32498: Handle bytes as input to urllib.parse.unquote Test ascii input. Simplified news item. --- Lib/test/test_urllib.py | 9 +++++++++ .../Library/2018-06-17-21-02-25.bpo-32498.La3TZz.rst | 4 ++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py index 0901fe3e7dd368f..3f59c660845938f 100644 --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@ -1209,12 +1209,21 @@ def test_unquote_with_unicode(self): "using unquote(): %r != %r" % (expect, result)) def test_unquoting_with_bytes_input(self): + # ASCII characters decoded to a string + given = b'blueberryjam' + expect = 'blueberryjam' + result = urllib.parse.unquote(given) + self.assertEqual(expect, result, + "using unquote(): %r != %r" % (expect, result)) + + # A mix of non-ASCII hex-encoded characters and ASCII characters given = b'bl\xc3\xa5b\xc3\xa6rsyltet\xc3\xb8y' expect = 'bl\u00e5b\u00e6rsyltet\u00f8y' result = urllib.parse.unquote(given) self.assertEqual(expect, result, "using unquote(): %r != %r" % (expect, result)) + # A mix of non-ASCII percent-encoded characters and ASCII characters given = b'bl%c3%a5b%c3%a6rsyltet%c3%b8j' expect = 'bl\u00e5b\u00e6rsyltet\u00f8j' result = urllib.parse.unquote(given) diff --git a/Misc/NEWS.d/next/Library/2018-06-17-21-02-25.bpo-32498.La3TZz.rst b/Misc/NEWS.d/next/Library/2018-06-17-21-02-25.bpo-32498.La3TZz.rst index ad2f9b7068723e6..56dc716f518ba3f 100644 --- a/Misc/NEWS.d/next/Library/2018-06-17-21-02-25.bpo-32498.La3TZz.rst +++ b/Misc/NEWS.d/next/Library/2018-06-17-21-02-25.bpo-32498.La3TZz.rst @@ -1,2 +1,2 @@ -Made ``urllib.parse.unquote()`` polymorphic, accepting both string and -bytes. Patch by Stein Karlsen. +Made ``urllib.parse.unquote()`` accepting both string and bytes. Patch by +Stein Karlsen. From 183993f83c901b580bf76399453b8d96cb6f7337 Mon Sep 17 00:00:00 2001 From: Stein Karlsen Date: Sun, 13 Oct 2019 20:01:09 +0200 Subject: [PATCH 7/8] bpo-32498: Handle bytes as input to urllib.parse.unquote Fixed typo in news item --- .../next/Library/2018-06-17-21-02-25.bpo-32498.La3TZz.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2018-06-17-21-02-25.bpo-32498.La3TZz.rst b/Misc/NEWS.d/next/Library/2018-06-17-21-02-25.bpo-32498.La3TZz.rst index 56dc716f518ba3f..57503ea96d943d8 100644 --- a/Misc/NEWS.d/next/Library/2018-06-17-21-02-25.bpo-32498.La3TZz.rst +++ b/Misc/NEWS.d/next/Library/2018-06-17-21-02-25.bpo-32498.La3TZz.rst @@ -1,2 +1,2 @@ -Made ``urllib.parse.unquote()`` accepting both string and bytes. Patch by +Made ``urllib.parse.unquote()`` accept both string and bytes. Patch by Stein Karlsen. From c01ad0dbf4034f623cab42cd2052b4ca63a7a2ea Mon Sep 17 00:00:00 2001 From: Tal Einat Date: Mon, 14 Oct 2019 13:13:02 +0300 Subject: [PATCH 8/8] reformat NEWS entry --- .../next/Library/2018-06-17-21-02-25.bpo-32498.La3TZz.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Misc/NEWS.d/next/Library/2018-06-17-21-02-25.bpo-32498.La3TZz.rst b/Misc/NEWS.d/next/Library/2018-06-17-21-02-25.bpo-32498.La3TZz.rst index 57503ea96d943d8..9df9e65e55b3eac 100644 --- a/Misc/NEWS.d/next/Library/2018-06-17-21-02-25.bpo-32498.La3TZz.rst +++ b/Misc/NEWS.d/next/Library/2018-06-17-21-02-25.bpo-32498.La3TZz.rst @@ -1,2 +1,2 @@ -Made ``urllib.parse.unquote()`` accept both string and bytes. Patch by -Stein Karlsen. +Made :func:`urllib.parse.unquote()` accept bytes in addition to strings. +Patch by Stein Karlsen.