From: Ben Darnell Date: Wed, 23 Feb 2011 22:03:49 +0000 (-0800) Subject: Move AsyncHTTPClient creation logic to httpclient.py. X-Git-Tag: v2.0.0~125 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=bfb171fb90ab5fc255e423a8cb3daf5514a36ac2;p=thirdparty%2Ftornado.git Move AsyncHTTPClient creation logic to httpclient.py. --- diff --git a/tornado/curl_httpclient.py b/tornado/curl_httpclient.py index c00f1945f..728c0b380 100644 --- a/tornado/curl_httpclient.py +++ b/tornado/curl_httpclient.py @@ -30,107 +30,57 @@ import pycurl import sys import threading import time -import weakref from tornado import httputil from tornado import ioloop from tornado import stack_context from tornado.escape import utf8 -from tornado.httpclient import HTTPRequest, HTTPResponse, HTTPError, main +from tornado.httpclient import HTTPRequest, HTTPResponse, HTTPError, AsyncHTTPClient, main + +class CurlAsyncHTTPClient(AsyncHTTPClient): + def initialize(self, io_loop=None, max_clients=10, + max_simultaneous_connections=None): + self.io_loop = io_loop + self._multi = pycurl.CurlMulti() + self._multi.setopt(pycurl.M_TIMERFUNCTION, self._set_timeout) + self._multi.setopt(pycurl.M_SOCKETFUNCTION, self._handle_socket) + self._curls = [_curl_create(max_simultaneous_connections) + for i in xrange(max_clients)] + self._free_list = self._curls[:] + self._requests = collections.deque() + self._fds = {} + self._timeout = None -class AsyncHTTPClient(object): - """An non-blocking HTTP client backed with pycurl. - - Example usage: - - import ioloop - - def handle_request(response): - if response.error: - print "Error:", response.error - else: - print response.body - ioloop.IOLoop.instance().stop() - - http_client = httpclient.AsyncHTTPClient() - http_client.fetch("http://www.google.com/", handle_request) - ioloop.IOLoop.instance().start() - - fetch() can take a string URL or an HTTPRequest instance, which offers - more options, like executing POST/PUT/DELETE requests. - - The keyword argument max_clients to the AsyncHTTPClient constructor - determines the maximum number of simultaneous fetch() operations that - can execute in parallel on each IOLoop. - """ - _ASYNC_CLIENTS = weakref.WeakKeyDictionary() - - def __new__(cls, io_loop=None, max_clients=10, - max_simultaneous_connections=None): - # There is one client per IOLoop since they share curl instances - io_loop = io_loop or ioloop.IOLoop.instance() - if io_loop in cls._ASYNC_CLIENTS: - return cls._ASYNC_CLIENTS[io_loop] - else: - instance = super(AsyncHTTPClient, cls).__new__(cls) - instance.io_loop = io_loop - instance._multi = pycurl.CurlMulti() - instance._multi.setopt(pycurl.M_TIMERFUNCTION, - instance._set_timeout) - instance._multi.setopt(pycurl.M_SOCKETFUNCTION, - instance._handle_socket) - instance._curls = [_curl_create(max_simultaneous_connections) - for i in xrange(max_clients)] - instance._free_list = instance._curls[:] - instance._requests = collections.deque() - instance._fds = {} - instance._timeout = None - cls._ASYNC_CLIENTS[io_loop] = instance - - try: - instance._socket_action = instance._multi.socket_action - except AttributeError: - # socket_action is found in pycurl since 7.18.2 (it's been - # in libcurl longer than that but wasn't accessible to - # python). - logging.warning("socket_action method missing from pycurl; " - "falling back to socket_all. Upgrading " - "libcurl and pycurl will improve performance") - instance._socket_action = \ - lambda fd, action: instance._multi.socket_all() - - # libcurl has bugs that sometimes cause it to not report all - # relevant file descriptors and timeouts to TIMERFUNCTION/ - # SOCKETFUNCTION. Mitigate the effects of such bugs by - # forcing a periodic scan of all active requests. - instance._force_timeout_callback = ioloop.PeriodicCallback( - instance._handle_force_timeout, 1000, io_loop=io_loop) - instance._force_timeout_callback.start() - - return instance + try: + self._socket_action = self._multi.socket_action + except AttributeError: + # socket_action is found in pycurl since 7.18.2 (it's been + # in libcurl longer than that but wasn't accessible to + # python). + logging.warning("socket_action method missing from pycurl; " + "falling back to socket_all. Upgrading " + "libcurl and pycurl will improve performance") + self._socket_action = \ + lambda fd, action: self._multi.socket_all() + + # libcurl has bugs that sometimes cause it to not report all + # relevant file descriptors and timeouts to TIMERFUNCTION/ + # SOCKETFUNCTION. Mitigate the effects of such bugs by + # forcing a periodic scan of all active requests. + self._force_timeout_callback = ioloop.PeriodicCallback( + self._handle_force_timeout, 1000, io_loop=io_loop) + self._force_timeout_callback.start() def close(self): - """Destroys this http client, freeing any file descriptors used. - Not needed in normal use, but may be helpful in unittests that - create and destroy http clients. No other methods may be called - on the AsyncHTTPClient after close(). - """ - del AsyncHTTPClient._ASYNC_CLIENTS[self.io_loop] self._force_timeout_callback.stop() for curl in self._curls: curl.close() self._multi.close() self._closed = True + super(CurlAsyncHTTPClient, self).close() def fetch(self, request, callback, **kwargs): - """Executes an HTTPRequest, calling callback with an HTTPResponse. - - If an error occurs during the fetch, the HTTPResponse given to the - callback has a non-None error attribute that contains the exception - encountered during the request. You can call response.rethrow() to - throw the exception (if any) in the callback. - """ if not isinstance(request, HTTPRequest): request = HTTPRequest(url=request, **kwargs) self._requests.append((request, stack_context.wrap(callback))) @@ -312,11 +262,6 @@ class AsyncHTTPClient(object): def handle_callback_exception(self, callback): self.io_loop.handle_callback_exception(callback) -# For backwards compatibility: Tornado 1.0 included a new implementation of -# AsyncHTTPClient that has since replaced the original. Define an alias -# so anything that used AsyncHTTPClient2 still works -AsyncHTTPClient2 = AsyncHTTPClient - class CurlError(HTTPError): def __init__(self, errno, message): diff --git a/tornado/httpclient.py b/tornado/httpclient.py index 39f7102ac..48faff2c5 100644 --- a/tornado/httpclient.py +++ b/tornado/httpclient.py @@ -1,6 +1,7 @@ import httplib import os import time +import weakref from tornado.escape import utf8 from tornado import httputil @@ -44,6 +45,66 @@ class HTTPClient(object): response.rethrow() return response +class AsyncHTTPClient(object): + """An non-blocking HTTP client. + + Example usage: + + import ioloop + + def handle_request(response): + if response.error: + print "Error:", response.error + else: + print response.body + ioloop.IOLoop.instance().stop() + + http_client = httpclient.AsyncHTTPClient() + http_client.fetch("http://www.google.com/", handle_request) + ioloop.IOLoop.instance().start() + + fetch() can take a string URL or an HTTPRequest instance, which offers + more options, like executing POST/PUT/DELETE requests. + + The keyword argument max_clients to the AsyncHTTPClient constructor + determines the maximum number of simultaneous fetch() operations that + can execute in parallel on each IOLoop. + """ + _ASYNC_CLIENTS = weakref.WeakKeyDictionary() + + def __new__(cls, io_loop=None, max_clients=10, force_instance=False, + **kwargs): + io_loop = io_loop or IOLoop.instance() + if io_loop in cls._ASYNC_CLIENTS and not force_instance: + return cls._ASYNC_CLIENTS[io_loop] + else: + if cls is AsyncHTTPClient: + cls = AsyncImpl + instance = super(AsyncHTTPClient, cls).__new__(cls) + instance.initialize(io_loop, max_clients, **kwargs) + if not force_instance: + cls._ASYNC_CLIENTS[io_loop] = instance + return instance + + def close(self): + """Destroys this http client, freeing any file descriptors used. + Not needed in normal use, but may be helpful in unittests that + create and destroy http clients. No other methods may be called + on the AsyncHTTPClient after close(). + """ + if self._ASYNC_CLIENTS[self.io_loop] is self: + del self._ASYNC_CLIENTS[self.io_loop] + + def fetch(self, request, callback, **kwargs): + """Executes an HTTPRequest, calling callback with an HTTPResponse. + + If an error occurs during the fetch, the HTTPResponse given to the + callback has a non-None error attribute that contains the exception + encountered during the request. You can call response.rethrow() to + throw the exception (if any) in the callback. + """ + raise NotImplementedError() + class HTTPRequest(object): def __init__(self, url, method="GET", headers=None, body=None, auth_username=None, auth_password=None, @@ -214,9 +275,9 @@ def main(): # and may be removed or replaced with a better way of specifying the preferred # HTTPClient implementation before the next release. if os.environ.get("USE_SIMPLE_HTTPCLIENT"): - from tornado.simple_httpclient import AsyncHTTPClient + from tornado.simple_httpclient import SimpleAsyncHTTPClient as AsyncImpl else: - from tornado.curl_httpclient import AsyncHTTPClient + from tornado.curl_httpclient import CurlAsyncHTTPClient as AsyncImpl if __name__ == "__main__": main() diff --git a/tornado/simple_httpclient.py b/tornado/simple_httpclient.py index 333d3efaf..38b5bc318 100644 --- a/tornado/simple_httpclient.py +++ b/tornado/simple_httpclient.py @@ -2,7 +2,7 @@ from __future__ import with_statement from cStringIO import StringIO -from tornado.httpclient import HTTPRequest, HTTPResponse, HTTPError +from tornado.httpclient import HTTPRequest, HTTPResponse, HTTPError, AsyncHTTPClient from tornado.httputil import HTTPHeaders from tornado.ioloop import IOLoop from tornado.iostream import IOStream, SSLIOStream @@ -19,7 +19,6 @@ import re import socket import time import urlparse -import weakref import zlib try: @@ -29,7 +28,7 @@ except ImportError: _DEFAULT_CA_CERTS = os.path.dirname(__file__) + '/ca-certificates.crt' -class AsyncHTTPClient(object): +class SimpleAsyncHTTPClient(AsyncHTTPClient): """Non-blocking HTTP client with no external dependencies. This class implements an HTTP 1.1 client on top of Tornado's IOStreams. @@ -55,12 +54,9 @@ class AsyncHTTPClient(object): should use the curl-based AsyncHTTPClient if HTTPS support is required. """ - _ASYNC_CLIENTS = weakref.WeakKeyDictionary() - - def __new__(cls, io_loop=None, max_clients=10, - max_simultaneous_connections=None, - force_instance=False, - hostname_mapping=None): + def initialize(self, io_loop=None, max_clients=10, + max_simultaneous_connections=None, + hostname_mapping=None): """Creates a AsyncHTTPClient. Only a single AsyncHTTPClient instance exists per IOLoop @@ -78,22 +74,11 @@ class AsyncHTTPClient(object): settings like /etc/hosts is not possible or desirable (e.g. in unittests). """ - io_loop = io_loop or IOLoop.instance() - if io_loop in cls._ASYNC_CLIENTS and not force_instance: - return cls._ASYNC_CLIENTS[io_loop] - else: - instance = super(AsyncHTTPClient, cls).__new__(cls) - instance.io_loop = io_loop - instance.max_clients = max_clients - instance.queue = collections.deque() - instance.active = {} - instance.hostname_mapping = hostname_mapping - if not force_instance: - cls._ASYNC_CLIENTS[io_loop] = instance - return instance - - def close(self): - pass + self.io_loop = io_loop + self.max_clients = max_clients + self.queue = collections.deque() + self.active = {} + self.hostname_mapping = hostname_mapping def fetch(self, request, callback, **kwargs): if not isinstance(request, HTTPRequest): @@ -393,16 +378,13 @@ def match_hostname(cert, hostname): raise CertificateError("no appropriate commonName or " "subjectAltName fields were found") -# Alias for backwards compatibility -SimpleAsyncHTTPClient = AsyncHTTPClient - def main(): from tornado.options import define, options, parse_command_line define("print_headers", type=bool, default=False) define("print_body", type=bool, default=True) define("follow_redirects", type=bool, default=True) args = parse_command_line() - client = AsyncHTTPClient() + client = SimpleAsyncHTTPClient() io_loop = IOLoop.instance() for arg in args: def callback(response): diff --git a/tornado/test/simple_httpclient_test.py b/tornado/test/simple_httpclient_test.py index ab797c435..55eb19120 100644 --- a/tornado/test/simple_httpclient_test.py +++ b/tornado/test/simple_httpclient_test.py @@ -76,7 +76,8 @@ class SimpleHTTPClientTestCase(AsyncHTTPTestCase, LogTrapTestCase): def setUp(self): super(SimpleHTTPClientTestCase, self).setUp() # replace the client defined in the parent class - self.http_client = SimpleAsyncHTTPClient(io_loop=self.io_loop) + self.http_client = SimpleAsyncHTTPClient(io_loop=self.io_loop, + force_instance=True) def test_hello_world(self): response = self.fetch("/hello")