From cdcf228f6ab764fcd8237231f0a473b9f80586ad Mon Sep 17 00:00:00 2001 From: Andrew James Date: Mon, 6 Jul 2026 01:57:09 +1000 Subject: [PATCH] gh-60055: Allow passing a Request instance for RobotParser URLs (#103753) MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit * Allow passing a Request instance for the url parameter * 📜🤖 Added by blurb_it. * gh-60055: rebase onto main and document Request support * Fix the lint warnings and CI Errors. * Addressing RobotFileParser review comments. * Update Doc/library/urllib.robotparser.rst Co-authored-by: Serhiy Storchaka * Address additional Review Comments. * Changed versionchanged to next. --------- Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> Co-authored-by: Senthil Kumaran Co-authored-by: Serhiy Storchaka --- Doc/library/urllib.robotparser.rst | 25 +++++++++++++-- Lib/test/test_robotparser.py | 31 +++++++++++++++++++ Lib/urllib/robotparser.py | 7 ++++- ...3-04-24-11-12-00.gh-issue-60055.UjP6aX.rst | 1 + 4 files changed, 61 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-04-24-11-12-00.gh-issue-60055.UjP6aX.rst diff --git a/Doc/library/urllib.robotparser.rst b/Doc/library/urllib.robotparser.rst index 1fa7fc13baa5..3e84f86d4dd3 100644 --- a/Doc/library/urllib.robotparser.rst +++ b/Doc/library/urllib.robotparser.rst @@ -24,11 +24,18 @@ structure of :file:`robots.txt` files, see :rfc:`9309`. .. class:: RobotFileParser(url='') This class provides methods to read, parse and answer questions about the - :file:`robots.txt` file at *url*. + :file:`robots.txt` file at *url* or a :class:`urllib.request.Request` object. + + .. versionchanged:: next + *url* parameter can be a :class:`urllib.request.Request` object. .. method:: set_url(url) - Sets the URL referring to a :file:`robots.txt` file. + Sets the URL referring to a :file:`robots.txt` file or a + :class:`urllib.request.Request` object. + + .. versionchanged:: next + *url* parameter can be a :class:`urllib.request.Request` object. .. method:: read() @@ -102,3 +109,17 @@ class:: True >>> rp.can_fetch("*", "http://www.pythontest.net/no-robots-here/") False + + +The following example demonstrates use of a :class:`urllib.request.Request` +object with additional user-agent headers populated:: + + >>> import urllib.robotparser + >>> import urllib.request + >>> rp = urllib.robotparser.RobotFileParser() + >>> rp.set_url(urllib.request.Request("http://www.pythontest.net/robots.txt", headers={"User-Agent": "IsraBot"})) + >>> rp.read() + >>> rp.can_fetch("*", "http://www.pythontest.net/") + True + >>> rp.can_fetch("*", "http://www.pythontest.net/no-robots-here/") + False diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index cd1477037e94..725c6e3b09e1 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -773,6 +773,37 @@ class HttpErrorsTestCase(BaseLocalNetworkTestCase, unittest.TestCase): self.assertFalse(parser.can_fetch("*", url + '/path/file.html')) +class UserAgentSiteTestCase(BaseLocalNetworkTestCase, unittest.TestCase): + + class RobotHandler(BaseHTTPRequestHandler): + def do_GET(self): + if self.headers.get('User-Agent').startswith('Python-urllib'): + self.send_error(403, "Forbidden access") + else: + self.send_response(200) + self.end_headers() + self.wfile.write(b"User-agent: *\nDisallow:") + + def log_message(self, format, *args): + pass + + def testUserAgentFilteringSite(self): + addr = self.server.server_address + url = f'http://{socket_helper.HOST}:{addr[1]}' + robots_url = url + "/robots.txt" + file_url = url + "/document" + parser = urllib.robotparser.RobotFileParser() + parser.set_url(robots_url) + parser.read() + self.assertTrue(parser.disallow_all) + self.assertFalse(parser.can_fetch("*", file_url)) + parser = urllib.robotparser.RobotFileParser() + parser.set_url(urllib.request.Request(robots_url, headers={'User-Agent': 'cybermapper'})) + parser.read() + self.assertFalse(parser.disallow_all) + self.assertTrue(parser.can_fetch("*", file_url)) + + @support.requires_working_socket() class NetworkTestCase(unittest.TestCase): diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py index 0c3e5d928909..61772d90e2d5 100644 --- a/Lib/urllib/robotparser.py +++ b/Lib/urllib/robotparser.py @@ -55,8 +55,13 @@ class RobotFileParser: self.last_checked = time.time() def set_url(self, url): - """Sets the URL referring to a robots.txt file.""" + """Sets the URL referring to a robots.txt file. + can be a string or a Request object. + """ self.url = url + + if isinstance(url, urllib.request.Request): + url = url.full_url self.host, self.path = urllib.parse.urlsplit(url)[1:3] def read(self): diff --git a/Misc/NEWS.d/next/Library/2023-04-24-11-12-00.gh-issue-60055.UjP6aX.rst b/Misc/NEWS.d/next/Library/2023-04-24-11-12-00.gh-issue-60055.UjP6aX.rst new file mode 100644 index 000000000000..58fface7cade --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-24-11-12-00.gh-issue-60055.UjP6aX.rst @@ -0,0 +1 @@ +Let ``urllib.robotparser.RobotFileParser`` accept a ``urllib.request.Request`` object as well as a url string when setting a robots.txt url. -- 2.47.3