#17403: urllib.parse.robotparser normalizes the urls before adding to ruleline.

This helps in handling certain types invalid urls in a conservative manner.

diff --git a/Lib/robotparser.py b/Lib/robotparser.py
index 1722863d144db7fc1956292892ebf49dc3f0cd3a..ad3be9471158b7dfaad6e1c66756332d39e137ac 100644 (file)
--- a/Lib/robotparser.py
+++ b/Lib/robotparser.py
@@ -160,6 +160,7 @@ class RuleLine:
         if path == '' and not allowance:
             # an empty value means allow all
             allowance = True
+        path = urlparse.urlunparse(urlparse.urlparse(path))
         self.path = urllib.quote(path)
         self.allowance = allowance
 

diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py
index b3d4a46056bb0d03d1d218c43b9185ee7daaea8b..651301bde2a35585e77c45b6b6b0eeb25e9ba114 100644 (file)
--- a/Lib/test/test_robotparser.py
+++ b/Lib/test/test_robotparser.py
@@ -228,6 +228,18 @@ bad = ['/some/path']
 
 RobotTest(15, doc, good, bad)
 
+# 16. Empty query (issue #17403). Normalizing the url first.
+doc = """
+User-agent: *
+Allow: /some/path?
+Disallow: /another/path?
+"""
+
+good = ['/some/path?']
+bad = ['/another/path?']
+
+RobotTest(16, doc, good, bad)
+
 
 class NetworkTestCase(unittest.TestCase):
 

diff --git a/Misc/NEWS b/Misc/NEWS
index 00448c45af4b93c487fe8c0152b245321eac5f2c..7e901f7006678aaeef8e63a3c5c531e96f353f98 100644 (file)
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -15,6 +15,10 @@ Core and Builtins
 Library
 -------
 
+- Issue #17403: urllib.parse.robotparser normalizes the urls before adding to
+  ruleline. This helps in handling certain types invalid urls in a conservative
+  manner. Patch contributed by Mher Movsisyan.
+
 - Implement inequality on weakref.WeakSet.
 
 - Issue #17981: Closed socket on error in SysLogHandler.