From 1a5b337552cf0f6062f2ac9ac401bbc8b2fc7b03 Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Sun, 18 Nov 2012 19:29:18 -0500 Subject: [PATCH] Allow default HTTPRequest attributes to be set globally via configure. Closes #379. --- tornado/curl_httpclient.py | 8 +++-- tornado/httpclient.py | 44 +++++++++++++++++++++++---- tornado/simple_httpclient.py | 13 +++++--- tornado/test/httpclient_test.py | 52 ++++++++++++++++++++++++++++++++ tornado/test/httpserver_test.py | 11 ++++--- website/sphinx/releases/next.rst | 10 ++++++ 6 files changed, 122 insertions(+), 16 deletions(-) diff --git a/tornado/curl_httpclient.py b/tornado/curl_httpclient.py index df3c7501b..a6c0bb0de 100644 --- a/tornado/curl_httpclient.py +++ b/tornado/curl_httpclient.py @@ -31,12 +31,15 @@ from tornado.log import gen_log from tornado import stack_context from tornado.escape import utf8 -from tornado.httpclient import HTTPRequest, HTTPResponse, HTTPError, AsyncHTTPClient, main +from tornado.httpclient import HTTPRequest, HTTPResponse, HTTPError, AsyncHTTPClient, main, _RequestProxy class CurlAsyncHTTPClient(AsyncHTTPClient): - def initialize(self, io_loop=None, max_clients=10): + def initialize(self, io_loop=None, max_clients=10, defaults=None): self.io_loop = io_loop + self.defaults = dict(HTTPRequest._DEFAULTS) + if defaults is not None: + self.defaults.update(defaults) self._multi = pycurl.CurlMulti() self._multi.setopt(pycurl.M_TIMERFUNCTION, self._set_timeout) self._multi.setopt(pycurl.M_SOCKETFUNCTION, self._handle_socket) @@ -77,6 +80,7 @@ class CurlAsyncHTTPClient(AsyncHTTPClient): def fetch(self, request, callback, **kwargs): if not isinstance(request, HTTPRequest): request = HTTPRequest(url=request, **kwargs) + request = _RequestProxy(request, self.defaults) self._requests.append((request, stack_context.wrap(callback))) self._process_queue() self._set_timeout(0) diff --git a/tornado/httpclient.py b/tornado/httpclient.py index 7ab4b046c..945e12b31 100644 --- a/tornado/httpclient.py +++ b/tornado/httpclient.py @@ -40,7 +40,7 @@ import weakref from tornado.escape import utf8 from tornado import httputil, stack_context from tornado.ioloop import IOLoop -from tornado.util import import_object, Configurable +from tornado.util import Configurable class HTTPClient(object): @@ -195,16 +195,30 @@ class AsyncHTTPClient(Configurable): class HTTPRequest(object): """HTTP client request object.""" + + # Default values for HTTPRequest parameters. + # Merged with the values on the request object by AsyncHTTPClient + # implementations. + _DEFAULTS = dict( + connect_timeout=20.0, + request_timeout=20.0, + follow_redirects=True, + max_redirects=5, + use_gzip=True, + proxy_password='', + allow_nonstandard_methods=False, + validate_cert=True) + def __init__(self, url, method="GET", headers=None, body=None, auth_username=None, auth_password=None, - connect_timeout=20.0, request_timeout=20.0, - if_modified_since=None, follow_redirects=True, - max_redirects=5, user_agent=None, use_gzip=True, + connect_timeout=None, request_timeout=None, + if_modified_since=None, follow_redirects=None, + max_redirects=None, user_agent=None, use_gzip=None, network_interface=None, streaming_callback=None, header_callback=None, prepare_curl_callback=None, proxy_host=None, proxy_port=None, proxy_username=None, - proxy_password='', allow_nonstandard_methods=False, - validate_cert=True, ca_certs=None, + proxy_password=None, allow_nonstandard_methods=None, + validate_cert=None, ca_certs=None, allow_ipv6=None, client_key=None, client_cert=None): """Creates an `HTTPRequest`. @@ -393,6 +407,24 @@ class HTTPError(Exception): self.response = response Exception.__init__(self, "HTTP %d: %s" % (self.code, message)) +class _RequestProxy(object): + """Combines an object with a dictionary of defaults. + + Used internally by AsyncHTTPClient implementations. + """ + def __init__(self, request, defaults): + self.request = request + self.defaults = defaults + + def __getattr__(self, name): + request_attr = getattr(self.request, name) + if request_attr is not None: + return request_attr + elif self.defaults is not None: + return self.defaults.get(name, None) + else: + return None + def main(): from tornado.options import define, options, parse_command_line diff --git a/tornado/simple_httpclient.py b/tornado/simple_httpclient.py index d9591485f..be5fb6737 100644 --- a/tornado/simple_httpclient.py +++ b/tornado/simple_httpclient.py @@ -2,7 +2,7 @@ from __future__ import absolute_import, division, with_statement from tornado.escape import utf8, _unicode, native_str -from tornado.httpclient import HTTPRequest, HTTPResponse, HTTPError, AsyncHTTPClient, main +from tornado.httpclient import HTTPRequest, HTTPResponse, HTTPError, AsyncHTTPClient, main, _RequestProxy from tornado.httputil import HTTPHeaders from tornado.iostream import IOStream, SSLIOStream from tornado.netutil import Resolver @@ -53,7 +53,7 @@ class SimpleAsyncHTTPClient(AsyncHTTPClient): """ def initialize(self, io_loop=None, max_clients=10, hostname_mapping=None, max_buffer_size=104857600, - resolver=None): + resolver=None, defaults=None): """Creates a AsyncHTTPClient. Only a single AsyncHTTPClient instance exists per IOLoop @@ -80,6 +80,9 @@ class SimpleAsyncHTTPClient(AsyncHTTPClient): self.hostname_mapping = hostname_mapping self.max_buffer_size = max_buffer_size self.resolver = resolver or Resolver(io_loop=io_loop) + self.defaults = dict(HTTPRequest._DEFAULTS) + if defaults is not None: + self.defaults.update(defaults) def fetch(self, request, callback, **kwargs): if not isinstance(request, HTTPRequest): @@ -88,6 +91,7 @@ class SimpleAsyncHTTPClient(AsyncHTTPClient): # so make sure we don't modify the caller's object. This is also # where normal dicts get converted to HTTPHeaders objects. request.headers = HTTPHeaders(request.headers) + request = _RequestProxy(request, self.defaults) callback = stack_context.wrap(callback) self.queue.append((request, callback)) self._process_queue() @@ -396,10 +400,11 @@ class _HTTPConnection(object): if (self.request.follow_redirects and self.request.max_redirects > 0 and self.code in (301, 302, 303, 307)): - new_request = copy.copy(self.request) + assert isinstance(self.request, _RequestProxy) + new_request = copy.copy(self.request.request) new_request.url = urlparse.urljoin(self.request.url, self.headers["Location"]) - new_request.max_redirects -= 1 + new_request.max_redirects = self.request.max_redirects - 1 del new_request.headers["Host"] # http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.4 # Client SHOULD make a GET request after a 303. diff --git a/tornado/test/httpclient_test.py b/tornado/test/httpclient_test.py index 5af168f97..88a858824 100644 --- a/tornado/test/httpclient_test.py +++ b/tornado/test/httpclient_test.py @@ -9,10 +9,12 @@ import functools import re from tornado.escape import utf8 +from tornado.httpclient import HTTPRequest, _RequestProxy from tornado.iostream import IOStream from tornado import netutil from tornado.stack_context import ExceptionStackContext from tornado.testing import AsyncHTTPTestCase, bind_unused_port +from tornado.test.util import unittest from tornado.util import b, bytes_type from tornado.web import Application, RequestHandler, url @@ -55,6 +57,12 @@ class EchoPostHandler(RequestHandler): def post(self): self.write(self.request.body) + +class UserAgentHandler(RequestHandler): + def get(self): + self.write(self.request.headers.get('User-Agent', 'User agent not set')) + + # These tests end up getting run redundantly: once here with the default # HTTPClient implementation, and then again in each implementation's own # test suite. @@ -69,6 +77,7 @@ class HTTPClientCommonTestCase(AsyncHTTPTestCase): url("/auth", AuthHandler), url("/countdown/([0-9]+)", CountdownHandler, name="countdown"), url("/echopost", EchoPostHandler), + url("/user_agent", UserAgentHandler), ], gzip=True) def test_hello_world(self): @@ -249,3 +258,46 @@ Transfer-Encoding: chunked self.fetch('/chunk', header_callback=header_callback) self.assertEqual(len(exc_info), 1) self.assertIs(exc_info[0][0], ZeroDivisionError) + + def test_configure_defaults(self): + defaults = dict(user_agent='TestDefaultUserAgent') + # Construct a new instance of the configured client class + client = self.http_client.__class__(self.io_loop, force_instance=True, + defaults=defaults) + client.fetch(self.get_url('/user_agent'), callback=self.stop) + response = self.wait() + self.assertEqual(response.body, b('TestDefaultUserAgent')) + + +class RequestProxyTest(unittest.TestCase): + def test_request_set(self): + proxy = _RequestProxy(HTTPRequest('http://example.com/', + user_agent='foo'), + dict()) + self.assertEqual(proxy.user_agent, 'foo') + + def test_default_set(self): + proxy = _RequestProxy(HTTPRequest('http://example.com/'), + dict(network_interface='foo')) + self.assertEqual(proxy.network_interface, 'foo') + + def test_both_set(self): + proxy = _RequestProxy(HTTPRequest('http://example.com/', + proxy_host='foo'), + dict(proxy_host='bar')) + self.assertEqual(proxy.proxy_host, 'foo') + + def test_neither_set(self): + proxy = _RequestProxy(HTTPRequest('http://example.com/'), + dict()) + self.assertIs(proxy.auth_username, None) + + def test_bad_attribute(self): + proxy = _RequestProxy(HTTPRequest('http://example.com/'), + dict()) + with self.assertRaises(AttributeError): + proxy.foo + + def test_defaults_none(self): + proxy = _RequestProxy(HTTPRequest('http://example.com/'), None) + self.assertIs(proxy.auth_username, None) diff --git a/tornado/test/httpserver_test.py b/tornado/test/httpserver_test.py index 3ced4e1b7..d0331240c 100644 --- a/tornado/test/httpserver_test.py +++ b/tornado/test/httpserver_test.py @@ -178,10 +178,13 @@ class HTTPConnectionTest(AsyncHTTPTestCase): def raw_fetch(self, headers, body): client = SimpleAsyncHTTPClient(self.io_loop) - conn = RawRequestHTTPConnection(self.io_loop, client, - httpclient.HTTPRequest(self.get_url("/")), - None, self.stop, - 1024 * 1024) + conn = RawRequestHTTPConnection( + self.io_loop, client, + httpclient._RequestProxy( + httpclient.HTTPRequest(self.get_url("/")), + dict(httpclient.HTTPRequest._DEFAULTS)), + None, self.stop, + 1024 * 1024) conn.set_request( b("\r\n").join(headers + [utf8("Content-Length: %d\r\n" % len(body))]) + diff --git a/website/sphinx/releases/next.rst b/website/sphinx/releases/next.rst index 6c6e6c67c..56befec7b 100644 --- a/website/sphinx/releases/next.rst +++ b/website/sphinx/releases/next.rst @@ -172,3 +172,13 @@ In progress * Secondary `AsyncHTTPClient` callbacks (``streaming_callback``, ``header_callback``, and ``prepare_curl_callback``) now respect `StackContext`. +* `AsyncHTTPClient.configure` and all `AsyncHTTPClient` constructors + now take a ``defaults`` keyword argument. This argument should be a + dictionary, and its values will be used in place of corresponding + attributes of `HTTPRequest` that are not set. +* All unset attributes of `tornado.httpclient.HTTPRequest` are now ``None``. + The default values of some attributes (``connect_timeout``, + ``request_timeout``, ``follow_redirects``, ``max_redirects``, + ``use_gzip``, ``proxy_password``, ``allow_nonstandard_methods``, + and ``validate_cert`` have been moved from `HTTPRequest` to the + client implementations. -- 2.47.2