From: Serhiy Storchaka Date: Sun, 5 Jul 2026 11:13:01 +0000 (+0300) Subject: gh-79638: Restore "Treat an unreachable robots.txt as disallow all" (GH-152525) X-Git-Url: http://git.ipfire.org/gitweb/index.cgi?a=commitdiff_plain;h=70100b9ea0b43a95226eee45652a5735bb91e9c7;p=thirdparty%2FPython%2Fcpython.git gh-79638: Restore "Treat an unreachable robots.txt as disallow all" (GH-152525) This change was accidentally reverted by f0daba1652c (gh-106693, GH-149514), which only intended to revert the ob_sval change. The tests were already restored by GH-149569. Co-authored-by: Claude Opus 4.8 (1M context) --- diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py index e70eae800367..0c3e5d928909 100644 --- a/Lib/urllib/robotparser.py +++ b/Lib/urllib/robotparser.py @@ -65,9 +65,17 @@ class RobotFileParser: f = urllib.request.urlopen(self.url) except urllib.error.HTTPError as err: if err.code in (401, 403): + # If access to robot.txt has the status Unauthorized/Forbidden, + # then most likely this applies to the entire site. self.disallow_all = True - elif err.code >= 400 and err.code < 500: + elif 400 <= err.code < 500: + # RFC 9309, Section 2.3.1.3: the crawler MAY access any + # resources on the server. self.allow_all = True + elif 500 <= err.code < 600: + # RFC 9309, Section 2.3.1.4: the crawler MUST assume + # complete disallow. + self.disallow_all = True err.close() else: raw = f.read() diff --git a/Misc/NEWS.d/next/Library/2025-09-05-20-50-35.gh-issue-79638.Y-JfaH.rst b/Misc/NEWS.d/next/Library/2025-09-05-20-50-35.gh-issue-79638.Y-JfaH.rst new file mode 100644 index 000000000000..bd9fff0bc2e3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-09-05-20-50-35.gh-issue-79638.Y-JfaH.rst @@ -0,0 +1,2 @@ +Disallow all access in :mod:`urllib.robotparser` if the ``robots.txt`` file +is unreachable due to server or network errors.