From: Skip Montanaro <skip@pobox.com>
Date: Tue, 28 Aug 2007 23:26:55 +0000 (+0000)
Subject: backport r57626
X-Git-Tag: v2.5.2c1~206
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=51cf11271f3e852edc807b255e781b018c498994;p=python

backport r57626
---

diff --git a/Lib/robotparser.py b/Lib/robotparser.py
index 48ea066682..cf0a58d6ee 100644
--- a/Lib/robotparser.py
+++ b/Lib/robotparser.py
@@ -233,6 +233,11 @@ class URLopener(urllib.FancyURLopener):
         urllib.FancyURLopener.__init__(self, *args)
         self.errcode = 200
 
+    def prompt_user_passwd(self, host, realm):
+        ## If robots.txt file is accessible only with a password,
+        ## we act as if the file wasn't there.
+        return None, None
+
     def http_error_default(self, url, fp, errcode, errmsg, headers):
         self.errcode = errcode
         return urllib.FancyURLopener.http_error_default(self, url, fp, errcode,
diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py
index 6a23b22db3..a8e1eaaaee 100644
--- a/Lib/test/test_robotparser.py
+++ b/Lib/test/test_robotparser.py
@@ -134,8 +134,19 @@ bad = [] # Bug report says "/" should be denied, but that is not in the RFC
 
 RobotTest(7, doc, good, bad)
 
+class TestCase(unittest.TestCase):
+    def runTest(self):
+        test_support.requires('network')
+        # whole site is password-protected.
+        url = 'http://mueblesmoraleda.com'
+        parser = robotparser.RobotFileParser()
+        parser.set_url(url)
+        parser.read()
+        self.assertEqual(parser.can_fetch("*", url+"/robots.txt"), False)
+
 def test_main():
     test_support.run_suite(tests)
+    TestCase().run()
 
 if __name__=='__main__':
     test_support.Verbose = 1