From: Miss Islington (bot) <31488909+miss-islington@users.noreply.github.com>
Date: Fri, 17 Jul 2026 06:23:28 +0000 (+0200)
Subject: [3.14] gh-153404: Silently ignore non-decimal digits in robots.txt Crawl-delay and... 
X-Git-Url: http://git.ipfire.org/index.cgi?a=commitdiff_plain;h=1b060a65467d62d6ed89a67ba5524f004fcef709;p=thirdparty%2FPython%2Fcpython.git

[3.14] gh-153404: Silently ignore non-decimal digits in robots.txt Crawl-delay and Request-rate (GH-153405) (GH-153831)

str.isdigit() returned True for non-decimal Unicode digits such as
U+00B2 SUPERSCRIPT TWO, which int() then rejected with ValueError.
(cherry picked from commit 597ec2d7e5f8bbd5a1eb9e6780444cae46cec3ea)

Co-authored-by: tonghuaroot (童话) <tonghuaroot@gmail.com>
---

diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py
index cd1477037e94..1ec64da064d5 100644
--- a/Lib/test/test_robotparser.py
+++ b/Lib/test/test_robotparser.py
@@ -188,6 +188,8 @@ class BaseRequestRateTest(BaseRobotTest):
                         parsed_request_rate.seconds,
                         self.request_rate.seconds
                     )
+                else:
+                    self.assertIsNone(parsed_request_rate)
 
 
 class EmptyFileTest(BaseRequestRateTest, unittest.TestCase):
@@ -246,6 +248,32 @@ Crawl-delay: pears
     bad = []
 
 
+class NonDecimalDigitsTest(BaseRequestRateTest, unittest.TestCase):
+    # Non-decimal Unicode digits pass str.isdigit() but int() rejects
+    # them, so the directive must be silently ignored, not raise.
+    robots_txt = """\
+User-Agent: *
+Disallow: /tmp/
+Crawl-delay: ²
+Request-rate: ²/5
+    """
+    good = ['/foo.html']
+    bad = ['/tmp/']
+    crawl_delay = None
+    request_rate = None
+
+
+class NonDecimalDenominatorTest(BaseRequestRateTest, unittest.TestCase):
+    robots_txt = """\
+User-agent: *
+Disallow: /tmp/
+Request-rate: 5/²
+    """
+    good = ['/foo.html']
+    request_rate = None
+    bad = ['/tmp/']
+
+
 class AnotherInvalidRequestRateTest(BaseRobotTest, unittest.TestCase):
     # also test that Allow and Diasallow works well with each other
     robots_txt = """\
diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py
index 0c3e5d928909..d267ed00345c 100644
--- a/Lib/urllib/robotparser.py
+++ b/Lib/urllib/robotparser.py
@@ -143,15 +143,15 @@ class RobotFileParser:
                         # before trying to convert to int we need to make
                         # sure that robots.txt has valid syntax otherwise
                         # it will crash
-                        if line[1].strip().isdigit():
+                        if line[1].strip().isdecimal():
                             entry.delay = int(line[1])
                         state = 2
                 elif line[0] == "request-rate":
                     if state != 0:
                         numbers = line[1].split('/')
                         # check if all values are sane
-                        if (len(numbers) == 2 and numbers[0].strip().isdigit()
-                            and numbers[1].strip().isdigit()):
+                        if (len(numbers) == 2 and numbers[0].strip().isdecimal()
+                            and numbers[1].strip().isdecimal()):
                             entry.req_rate = RequestRate(int(numbers[0]), int(numbers[1]))
                         state = 2
                 elif line[0] == "sitemap":
diff --git a/Misc/NEWS.d/next/Library/2026-07-09-16-35-47.gh-issue-153404.daRkes.rst b/Misc/NEWS.d/next/Library/2026-07-09-16-35-47.gh-issue-153404.daRkes.rst
new file mode 100644
index 000000000000..0746d3f12149
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2026-07-09-16-35-47.gh-issue-153404.daRkes.rst
@@ -0,0 +1,4 @@
+:class:`urllib.robotparser.RobotFileParser` now silently ignores a
+``Crawl-delay`` or ``Request-rate`` value written with non-decimal digits
+(such as ``U+00B2 SUPERSCRIPT TWO``) instead of raising :exc:`ValueError`
+and aborting the parse of the whole ``robots.txt`` file.