From f2daee5d1222e51080d9236adbf9df696251fa33 Mon Sep 17 00:00:00 2001
From: fuwaraido <55026212+fuwaraido@users.noreply.github.com>
Date: Sun, 8 Sep 2019 00:53:41 +0900
Subject: [PATCH] some website requests User-Agent field in headers

I found some website like "sony.co.jp" requests User-Agent field in headers when I try to fetch their robots.txt. Unless they returns 403 Forbidden status code and RobotTextParser fails to read robots.txt.  Currently there is no way to specify headers so I propose add headers arguments in constructor.
---
 Lib/urllib/robotparser.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py
index f3bd806f072683d..cb80347bf2d1f47 100644
--- a/Lib/urllib/robotparser.py
+++ b/Lib/urllib/robotparser.py
@@ -25,13 +25,14 @@ class RobotFileParser:
 
     """
 
-    def __init__(self, url=''):
+    def __init__(self, url='', headers={}):
         self.entries = []
         self.default_entry = None
         self.disallow_all = False
         self.allow_all = False
         self.set_url(url)
         self.last_checked = 0
+        self.headers = headers
 
     def mtime(self):
         """Returns the time the robots.txt file was last fetched.
@@ -58,7 +59,10 @@ def set_url(self, url):
     def read(self):
         """Reads the robots.txt URL and feeds it to the parser."""
         try:
-            f = urllib.request.urlopen(self.url)
+            req = urllib.request.Request(self.url)
+            for k in self.headers.keys():
+                req.add_header(k, self.headers[k])
+            f = urllib.request.urlopen(req)
         except urllib.error.HTTPError as err:
             if err.code in (401, 403):
                 self.disallow_all = True