From 51cf11271f3e852edc807b255e781b018c498994 Mon Sep 17 00:00:00 2001 From: Skip Montanaro Date: Tue, 28 Aug 2007 23:26:55 +0000 Subject: [PATCH] backport r57626 --- Lib/robotparser.py | 5 +++++ Lib/test/test_robotparser.py | 11 +++++++++++ 2 files changed, 16 insertions(+) diff --git a/Lib/robotparser.py b/Lib/robotparser.py index 48ea066682e1..cf0a58d6ee32 100644 --- a/Lib/robotparser.py +++ b/Lib/robotparser.py @@ -233,6 +233,11 @@ class URLopener(urllib.FancyURLopener): urllib.FancyURLopener.__init__(self, *args) self.errcode = 200 + def prompt_user_passwd(self, host, realm): + ## If robots.txt file is accessible only with a password, + ## we act as if the file wasn't there. + return None, None + def http_error_default(self, url, fp, errcode, errmsg, headers): self.errcode = errcode return urllib.FancyURLopener.http_error_default(self, url, fp, errcode, diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index 6a23b22db3c0..a8e1eaaaeeee 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -134,8 +134,19 @@ bad = [] # Bug report says "/" should be denied, but that is not in the RFC RobotTest(7, doc, good, bad) +class TestCase(unittest.TestCase): + def runTest(self): + test_support.requires('network') + # whole site is password-protected. + url = 'http://mueblesmoraleda.com' + parser = robotparser.RobotFileParser() + parser.set_url(url) + parser.read() + self.assertEqual(parser.can_fetch("*", url+"/robots.txt"), False) + def test_main(): test_support.run_suite(tests) + TestCase().run() if __name__=='__main__': test_support.Verbose = 1 -- 2.47.3