From: Senthil Kumaran Date: Sun, 20 Dec 2009 06:05:13 +0000 (+0000) Subject: Fix for issue 7291 - urllib2 cannot handle https with proxy requiring auth X-Git-Tag: v2.7a2~151 X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=7713acf201e9638966a9a8f8e38446400410e826;p=python Fix for issue 7291 - urllib2 cannot handle https with proxy requiring auth Refactored HTTPHandler tests and added testcase for proxy authorization. --- diff --git a/Lib/httplib.py b/Lib/httplib.py index 3f44f12844..39acd1c04b 100644 --- a/Lib/httplib.py +++ b/Lib/httplib.py @@ -667,15 +667,24 @@ class HTTPConnection: self._method = None self._tunnel_host = None self._tunnel_port = None + self._tunnel_headers = {} self._set_hostport(host, port) if strict is not None: self.strict = strict - def set_tunnel(self, host, port=None): - """ Sets up the host and the port for the HTTP CONNECT Tunnelling.""" + def set_tunnel(self, host, port=None, headers=None): + """ Sets up the host and the port for the HTTP CONNECT Tunnelling. + + The headers argument should be a mapping of extra HTTP headers + to send with the CONNECT request. + """ self._tunnel_host = host self._tunnel_port = port + if headers: + self._tunnel_headers = headers + else: + self._tunnel_headers.clear() def _set_hostport(self, host, port): if port is None: @@ -699,7 +708,10 @@ class HTTPConnection: def _tunnel(self): self._set_hostport(self._tunnel_host, self._tunnel_port) - self.send("CONNECT %s:%d HTTP/1.0\r\n\r\n" % (self.host, self.port)) + self.send("CONNECT %s:%d HTTP/1.0\r\n" % (self.host, self.port)) + for header, value in self._tunnel_headers.iteritems(): + self.send("%s: %s\r\n" % (header, value)) + self.send("\r\n") response = self.response_class(self.sock, strict = self.strict, method = self._method) (version, code, message) = response._read_status() diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index e04d4a0412..01c214a252 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -261,6 +261,50 @@ class FakeMethod: def __call__(self, *args): return self.handle(self.meth_name, self.action, *args) +class MockHTTPResponse: + def __init__(self, fp, msg, status, reason): + self.fp = fp + self.msg = msg + self.status = status + self.reason = reason + def read(self): + return '' + +class MockHTTPClass: + def __init__(self): + self.req_headers = [] + self.data = None + self.raise_on_endheaders = False + self._tunnel_headers = {} + + def __call__(self, host, timeout=socket._GLOBAL_DEFAULT_TIMEOUT): + self.host = host + self.timeout = timeout + return self + + def set_debuglevel(self, level): + self.level = level + + def set_tunnel(self, host, port=None, headers=None): + self._tunnel_host = host + self._tunnel_port = port + if headers: + self._tunnel_headers = headers + else: + self._tunnel_headers.clear() + def request(self, method, url, body=None, headers={}): + self.method = method + self.selector = url + self.req_headers += headers.items() + self.req_headers.sort() + if body: + self.data = body + if self.raise_on_endheaders: + import socket + raise socket.error() + def getresponse(self): + return MockHTTPResponse(MockFile(), {}, 200, "OK") + class MockHandler: # useful for testing handler machinery # see add_ordered_mock_handlers() docstring @@ -368,6 +412,13 @@ class MockHTTPHandler(urllib2.BaseHandler): msg = mimetools.Message(StringIO("\r\n\r\n")) return MockResponse(200, "OK", msg, "", req.get_full_url()) +class MockHTTPSHandler(urllib2.AbstractHTTPHandler): + # Useful for testing the Proxy-Authorization request by verifying the + # properties of httpcon + httpconn = MockHTTPClass() + def https_open(self, req): + return self.do_open(self.httpconn, req) + class MockPasswordManager: def add_password(self, realm, uri, user, password): self.realm = realm @@ -680,37 +731,6 @@ class HandlerTests(unittest.TestCase): self.assertEqual(req.type, "ftp") def test_http(self): - class MockHTTPResponse: - def __init__(self, fp, msg, status, reason): - self.fp = fp - self.msg = msg - self.status = status - self.reason = reason - def read(self): - return '' - class MockHTTPClass: - def __init__(self): - self.req_headers = [] - self.data = None - self.raise_on_endheaders = False - def __call__(self, host, timeout=socket._GLOBAL_DEFAULT_TIMEOUT): - self.host = host - self.timeout = timeout - return self - def set_debuglevel(self, level): - self.level = level - def request(self, method, url, body=None, headers={}): - self.method = method - self.selector = url - self.req_headers += headers.items() - self.req_headers.sort() - if body: - self.data = body - if self.raise_on_endheaders: - import socket - raise socket.error() - def getresponse(self): - return MockHTTPResponse(MockFile(), {}, 200, "OK") h = urllib2.AbstractHTTPHandler() o = h.parent = MockOpener() @@ -974,6 +994,29 @@ class HandlerTests(unittest.TestCase): self.assertEqual([(handlers[0], "https_open")], [tup[0:2] for tup in o.calls]) + def test_proxy_https_proxy_authorization(self): + o = OpenerDirector() + ph = urllib2.ProxyHandler(dict(https='proxy.example.com:3128')) + o.add_handler(ph) + https_handler = MockHTTPSHandler() + o.add_handler(https_handler) + req = Request("https://www.example.com/") + req.add_header("Proxy-Authorization","FooBar") + req.add_header("User-Agent","Grail") + self.assertEqual(req.get_host(), "www.example.com") + self.assertIsNone(req._tunnel_host) + r = o.open(req) + # Verify Proxy-Authorization gets tunneled to request. + # httpsconn req_headers do not have the Proxy-Authorization header but + # the req will have. + self.assertFalse(("Proxy-Authorization","FooBar") in + https_handler.httpconn.req_headers) + self.assertTrue(("User-Agent","Grail") in + https_handler.httpconn.req_headers) + self.assertIsNotNone(req._tunnel_host) + self.assertEqual(req.get_host(), "proxy.example.com:3128") + self.assertEqual(req.get_header("Proxy-authorization"),"FooBar") + def test_basic_auth(self, quote_char='"'): opener = OpenerDirector() password_manager = MockPasswordManager() diff --git a/Lib/urllib2.py b/Lib/urllib2.py index 8dcf8dad58..0f59096898 100644 --- a/Lib/urllib2.py +++ b/Lib/urllib2.py @@ -1120,7 +1120,14 @@ class AbstractHTTPHandler(BaseHandler): (name.title(), val) for name, val in headers.items()) if req._tunnel_host: - h.set_tunnel(req._tunnel_host) + tunnel_headers = {} + proxy_auth_hdr = "Proxy-Authorization" + if proxy_auth_hdr in headers: + tunnel_headers[proxy_auth_hdr] = headers[proxy_auth_hdr] + # Proxy-Authorization should not be sent to origin + # server. + del headers[proxy_auth_hdr] + h.set_tunnel(req._tunnel_host, headers=tunnel_headers) try: h.request(req.get_method(), req.get_selector(), req.data, headers) diff --git a/Misc/NEWS b/Misc/NEWS index a326c2939c..c5d2380f82 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -24,6 +24,9 @@ Core and Builtins Library ------- +- Issue #7231: urllib2 cannot handle https with proxy requiring auth. + Patch by Tatsuhiro Tsujikawa. + - Issue #7349: Make methods of file objects in the io module accept None as an argument where file-like objects (ie StringIO and BytesIO) accept them to mean the same as passing no argument.