Closes #379.
from tornado import stack_context
from tornado.escape import utf8
-from tornado.httpclient import HTTPRequest, HTTPResponse, HTTPError, AsyncHTTPClient, main
+from tornado.httpclient import HTTPRequest, HTTPResponse, HTTPError, AsyncHTTPClient, main, _RequestProxy
class CurlAsyncHTTPClient(AsyncHTTPClient):
- def initialize(self, io_loop=None, max_clients=10):
+ def initialize(self, io_loop=None, max_clients=10, defaults=None):
self.io_loop = io_loop
+ self.defaults = dict(HTTPRequest._DEFAULTS)
+ if defaults is not None:
+ self.defaults.update(defaults)
self._multi = pycurl.CurlMulti()
self._multi.setopt(pycurl.M_TIMERFUNCTION, self._set_timeout)
self._multi.setopt(pycurl.M_SOCKETFUNCTION, self._handle_socket)
def fetch(self, request, callback, **kwargs):
if not isinstance(request, HTTPRequest):
request = HTTPRequest(url=request, **kwargs)
+ request = _RequestProxy(request, self.defaults)
self._requests.append((request, stack_context.wrap(callback)))
self._process_queue()
self._set_timeout(0)
from tornado.escape import utf8
from tornado import httputil, stack_context
from tornado.ioloop import IOLoop
-from tornado.util import import_object, Configurable
+from tornado.util import Configurable
class HTTPClient(object):
class HTTPRequest(object):
"""HTTP client request object."""
+
+ # Default values for HTTPRequest parameters.
+ # Merged with the values on the request object by AsyncHTTPClient
+ # implementations.
+ _DEFAULTS = dict(
+ connect_timeout=20.0,
+ request_timeout=20.0,
+ follow_redirects=True,
+ max_redirects=5,
+ use_gzip=True,
+ proxy_password='',
+ allow_nonstandard_methods=False,
+ validate_cert=True)
+
def __init__(self, url, method="GET", headers=None, body=None,
auth_username=None, auth_password=None,
- connect_timeout=20.0, request_timeout=20.0,
- if_modified_since=None, follow_redirects=True,
- max_redirects=5, user_agent=None, use_gzip=True,
+ connect_timeout=None, request_timeout=None,
+ if_modified_since=None, follow_redirects=None,
+ max_redirects=None, user_agent=None, use_gzip=None,
network_interface=None, streaming_callback=None,
header_callback=None, prepare_curl_callback=None,
proxy_host=None, proxy_port=None, proxy_username=None,
- proxy_password='', allow_nonstandard_methods=False,
- validate_cert=True, ca_certs=None,
+ proxy_password=None, allow_nonstandard_methods=None,
+ validate_cert=None, ca_certs=None,
allow_ipv6=None,
client_key=None, client_cert=None):
"""Creates an `HTTPRequest`.
self.response = response
Exception.__init__(self, "HTTP %d: %s" % (self.code, message))
+class _RequestProxy(object):
+ """Combines an object with a dictionary of defaults.
+
+ Used internally by AsyncHTTPClient implementations.
+ """
+ def __init__(self, request, defaults):
+ self.request = request
+ self.defaults = defaults
+
+ def __getattr__(self, name):
+ request_attr = getattr(self.request, name)
+ if request_attr is not None:
+ return request_attr
+ elif self.defaults is not None:
+ return self.defaults.get(name, None)
+ else:
+ return None
+
def main():
from tornado.options import define, options, parse_command_line
from __future__ import absolute_import, division, with_statement
from tornado.escape import utf8, _unicode, native_str
-from tornado.httpclient import HTTPRequest, HTTPResponse, HTTPError, AsyncHTTPClient, main
+from tornado.httpclient import HTTPRequest, HTTPResponse, HTTPError, AsyncHTTPClient, main, _RequestProxy
from tornado.httputil import HTTPHeaders
from tornado.iostream import IOStream, SSLIOStream
from tornado.netutil import Resolver
"""
def initialize(self, io_loop=None, max_clients=10,
hostname_mapping=None, max_buffer_size=104857600,
- resolver=None):
+ resolver=None, defaults=None):
"""Creates a AsyncHTTPClient.
Only a single AsyncHTTPClient instance exists per IOLoop
self.hostname_mapping = hostname_mapping
self.max_buffer_size = max_buffer_size
self.resolver = resolver or Resolver(io_loop=io_loop)
+ self.defaults = dict(HTTPRequest._DEFAULTS)
+ if defaults is not None:
+ self.defaults.update(defaults)
def fetch(self, request, callback, **kwargs):
if not isinstance(request, HTTPRequest):
# so make sure we don't modify the caller's object. This is also
# where normal dicts get converted to HTTPHeaders objects.
request.headers = HTTPHeaders(request.headers)
+ request = _RequestProxy(request, self.defaults)
callback = stack_context.wrap(callback)
self.queue.append((request, callback))
self._process_queue()
if (self.request.follow_redirects and
self.request.max_redirects > 0 and
self.code in (301, 302, 303, 307)):
- new_request = copy.copy(self.request)
+ assert isinstance(self.request, _RequestProxy)
+ new_request = copy.copy(self.request.request)
new_request.url = urlparse.urljoin(self.request.url,
self.headers["Location"])
- new_request.max_redirects -= 1
+ new_request.max_redirects = self.request.max_redirects - 1
del new_request.headers["Host"]
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.4
# Client SHOULD make a GET request after a 303.
import re
from tornado.escape import utf8
+from tornado.httpclient import HTTPRequest, _RequestProxy
from tornado.iostream import IOStream
from tornado import netutil
from tornado.stack_context import ExceptionStackContext
from tornado.testing import AsyncHTTPTestCase, bind_unused_port
+from tornado.test.util import unittest
from tornado.util import b, bytes_type
from tornado.web import Application, RequestHandler, url
def post(self):
self.write(self.request.body)
+
+class UserAgentHandler(RequestHandler):
+ def get(self):
+ self.write(self.request.headers.get('User-Agent', 'User agent not set'))
+
+
# These tests end up getting run redundantly: once here with the default
# HTTPClient implementation, and then again in each implementation's own
# test suite.
url("/auth", AuthHandler),
url("/countdown/([0-9]+)", CountdownHandler, name="countdown"),
url("/echopost", EchoPostHandler),
+ url("/user_agent", UserAgentHandler),
], gzip=True)
def test_hello_world(self):
self.fetch('/chunk', header_callback=header_callback)
self.assertEqual(len(exc_info), 1)
self.assertIs(exc_info[0][0], ZeroDivisionError)
+
+ def test_configure_defaults(self):
+ defaults = dict(user_agent='TestDefaultUserAgent')
+ # Construct a new instance of the configured client class
+ client = self.http_client.__class__(self.io_loop, force_instance=True,
+ defaults=defaults)
+ client.fetch(self.get_url('/user_agent'), callback=self.stop)
+ response = self.wait()
+ self.assertEqual(response.body, b('TestDefaultUserAgent'))
+
+
+class RequestProxyTest(unittest.TestCase):
+ def test_request_set(self):
+ proxy = _RequestProxy(HTTPRequest('http://example.com/',
+ user_agent='foo'),
+ dict())
+ self.assertEqual(proxy.user_agent, 'foo')
+
+ def test_default_set(self):
+ proxy = _RequestProxy(HTTPRequest('http://example.com/'),
+ dict(network_interface='foo'))
+ self.assertEqual(proxy.network_interface, 'foo')
+
+ def test_both_set(self):
+ proxy = _RequestProxy(HTTPRequest('http://example.com/',
+ proxy_host='foo'),
+ dict(proxy_host='bar'))
+ self.assertEqual(proxy.proxy_host, 'foo')
+
+ def test_neither_set(self):
+ proxy = _RequestProxy(HTTPRequest('http://example.com/'),
+ dict())
+ self.assertIs(proxy.auth_username, None)
+
+ def test_bad_attribute(self):
+ proxy = _RequestProxy(HTTPRequest('http://example.com/'),
+ dict())
+ with self.assertRaises(AttributeError):
+ proxy.foo
+
+ def test_defaults_none(self):
+ proxy = _RequestProxy(HTTPRequest('http://example.com/'), None)
+ self.assertIs(proxy.auth_username, None)
def raw_fetch(self, headers, body):
client = SimpleAsyncHTTPClient(self.io_loop)
- conn = RawRequestHTTPConnection(self.io_loop, client,
- httpclient.HTTPRequest(self.get_url("/")),
- None, self.stop,
- 1024 * 1024)
+ conn = RawRequestHTTPConnection(
+ self.io_loop, client,
+ httpclient._RequestProxy(
+ httpclient.HTTPRequest(self.get_url("/")),
+ dict(httpclient.HTTPRequest._DEFAULTS)),
+ None, self.stop,
+ 1024 * 1024)
conn.set_request(
b("\r\n").join(headers +
[utf8("Content-Length: %d\r\n" % len(body))]) +
* Secondary `AsyncHTTPClient` callbacks (``streaming_callback``,
``header_callback``, and ``prepare_curl_callback``) now respect
`StackContext`.
+* `AsyncHTTPClient.configure` and all `AsyncHTTPClient` constructors
+ now take a ``defaults`` keyword argument. This argument should be a
+ dictionary, and its values will be used in place of corresponding
+ attributes of `HTTPRequest` that are not set.
+* All unset attributes of `tornado.httpclient.HTTPRequest` are now ``None``.
+ The default values of some attributes (``connect_timeout``,
+ ``request_timeout``, ``follow_redirects``, ``max_redirects``,
+ ``use_gzip``, ``proxy_password``, ``allow_nonstandard_methods``,
+ and ``validate_cert`` have been moved from `HTTPRequest` to the
+ client implementations.
projects / thirdparty / tornado.git / commitdiff
