From: Ben Darnell Date: Mon, 18 Feb 2013 04:12:15 +0000 (-0500) Subject: Make Resolver Configurable and test with a threaded resolver. X-Git-Tag: v3.0.0~108 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=21acd099a7597ada52d0e516eddde69f2f037799;p=thirdparty%2Ftornado.git Make Resolver Configurable and test with a threaded resolver. --- diff --git a/tornado/netutil.py b/tornado/netutil.py index 05d8468ba..4003245e3 100644 --- a/tornado/netutil.py +++ b/tornado/netutil.py @@ -28,6 +28,7 @@ import stat from tornado.concurrent import dummy_executor, run_on_executor from tornado.ioloop import IOLoop from tornado.platform.auto import set_close_exec +from tornado.util import Configurable def bind_sockets(port, address=None, family=socket.AF_UNSPEC, backlog=128, flags=None): @@ -141,7 +142,15 @@ def add_accept_handler(sock, callback, io_loop=None): io_loop.add_handler(sock.fileno(), accept_handler, IOLoop.READ) -class BaseResolver(object): +class Resolver(Configurable): + @classmethod + def configurable_base(cls): + return Resolver + + @classmethod + def configurable_default(cls): + return BlockingResolver + def getaddrinfo(self, *args, **kwargs): """Resolves an address. @@ -157,8 +166,8 @@ class BaseResolver(object): raise NotImplementedError() -class Resolver(BaseResolver): - def __init__(self, io_loop=None, executor=None): +class ExecutorResolver(Resolver): + def initialize(self, io_loop=None, executor=None): self.io_loop = io_loop or IOLoop.instance() self.executor = executor or dummy_executor @@ -166,7 +175,17 @@ class Resolver(BaseResolver): def getaddrinfo(self, *args, **kwargs): return socket.getaddrinfo(*args, **kwargs) -class OverrideResolver(BaseResolver): +class BlockingResolver(ExecutorResolver): + def initialize(self, io_loop=None): + super(BlockingResolver, self).initialize(io_loop=io_loop) + +class ThreadedResolver(ExecutorResolver): + def initialize(self, io_loop=None, num_threads=10): + from concurrent.futures import ThreadPoolExecutor + super(ThreadedResolver, self).initialize( + io_loop=io_loop, executor=ThreadPoolExecutor(num_threads)) + +class OverrideResolver(Resolver): """Wraps a resolver with a mapping of overrides. This can be used to make local DNS changes (e.g. for testing) @@ -174,7 +193,7 @@ class OverrideResolver(BaseResolver): The mapping can contain either host strings or host-port pairs. """ - def __init__(self, resolver, mapping): + def initialize(self, resolver, mapping): self.resolver = resolver self.mapping = mapping diff --git a/tornado/simple_httpclient.py b/tornado/simple_httpclient.py index 73739ff40..d659318da 100644 --- a/tornado/simple_httpclient.py +++ b/tornado/simple_httpclient.py @@ -75,7 +75,8 @@ class SimpleAsyncHTTPClient(AsyncHTTPClient): self.max_buffer_size = max_buffer_size self.resolver = resolver or Resolver(io_loop=io_loop) if hostname_mapping is not None: - self.resolver = OverrideResolver(self.resolver, hostname_mapping) + self.resolver = OverrideResolver(resolver=self.resolver, + mapping=hostname_mapping) def fetch_impl(self, request, callback): self.queue.append((request, callback)) diff --git a/tornado/test/httpserver_test.py b/tornado/test/httpserver_test.py index 87a3e3e83..2a60e9de1 100644 --- a/tornado/test/httpserver_test.py +++ b/tornado/test/httpserver_test.py @@ -190,7 +190,7 @@ class HTTPConnectionTest(AsyncHTTPTestCase): httpclient.HTTPRequest(self.get_url("/")), dict(httpclient.HTTPRequest._DEFAULTS)), None, self.stop, - 1024 * 1024, Resolver(self.io_loop)) + 1024 * 1024, Resolver(io_loop=self.io_loop)) conn.set_request( b"\r\n".join(headers + [utf8("Content-Length: %d\r\n" % len(body))]) + diff --git a/tornado/test/netutil_test.py b/tornado/test/netutil_test.py index 4342e18e3..4711019fa 100644 --- a/tornado/test/netutil_test.py +++ b/tornado/test/netutil_test.py @@ -2,7 +2,7 @@ from __future__ import absolute_import, division, print_function, with_statement import socket -from tornado.netutil import Resolver +from tornado.netutil import BlockingResolver, ThreadedResolver from tornado.testing import AsyncTestCase from tornado.test.util import unittest @@ -28,20 +28,18 @@ class _ResolverTestMixin(object): future.result()) -class SyncResolverTest(AsyncTestCase, _ResolverTestMixin): +class BlockingResolverTest(AsyncTestCase, _ResolverTestMixin): def setUp(self): - super(SyncResolverTest, self).setUp() - self.resolver = Resolver(self.io_loop) + super(BlockingResolverTest, self).setUp() + self.resolver = BlockingResolver(io_loop=self.io_loop) @unittest.skipIf(futures is None, "futures module not present") class ThreadedResolverTest(AsyncTestCase, _ResolverTestMixin): def setUp(self): super(ThreadedResolverTest, self).setUp() - from concurrent.futures import ThreadPoolExecutor - self.executor = ThreadPoolExecutor(2) - self.resolver = Resolver(self.io_loop, self.executor) + self.resolver = ThreadedResolver(io_loop=self.io_loop) def tearDown(self): - self.executor.shutdown() + self.resolver.executor.shutdown() super(ThreadedResolverTest, self).tearDown() diff --git a/tornado/test/runtests.py b/tornado/test/runtests.py index 24717114e..e9e21bda8 100644 --- a/tornado/test/runtests.py +++ b/tornado/test/runtests.py @@ -6,6 +6,7 @@ import textwrap import sys from tornado.httpclient import AsyncHTTPClient from tornado.ioloop import IOLoop +from tornado.netutil import Resolver from tornado.options import define, options, add_parse_callback from tornado.test.util import unittest @@ -85,6 +86,8 @@ if __name__ == '__main__': callback=AsyncHTTPClient.configure) define('ioloop', type=str, default=None) define('ioloop_time_monotonic', default=False) + define('resolver', type=str, default=None, + callback=Resolver.configure) def configure_ioloop(): kwargs = {} diff --git a/tornado/websocket.py b/tornado/websocket.py index 3c914d99b..235a1102e 100644 --- a/tornado/websocket.py +++ b/tornado/websocket.py @@ -746,7 +746,7 @@ class _WebSocketClientConnection(simple_httpclient._HTTPConnection): super(_WebSocketClientConnection, self).__init__( io_loop, None, request, lambda: None, lambda response: None, - 104857600, Resolver(io_loop)) + 104857600, Resolver(io_loop=io_loop)) def _on_close(self): self.on_message(None) diff --git a/tox.ini b/tox.ini index cfbf468df..bc0f73a67 100644 --- a/tox.ini +++ b/tox.ini @@ -11,7 +11,7 @@ [tox] # "-full" variants include optional dependencies, to ensure # that things work both in a bare install and with all the extras. -envlist = py27-full, py27-curl, py32-full, pypy, py26, py26-full, py27, py32, py32-utf8, py33, py27-opt, py32-opt, pypy-full, py27-select, py27-monotonic, py33-monotonic, py27-twisted +envlist = py27-full, py27-curl, py32-full, pypy, py26, py26-full, py27, py32, py32-utf8, py33, py27-opt, py32-opt, pypy-full, py27-select, py27-monotonic, py33-monotonic, py27-twisted, py27-threadedresolver [testenv] commands = python -m tornado.test.runtests {posargs:} @@ -91,6 +91,15 @@ deps = twisted commands = python -m tornado.test.runtests --ioloop_time_monotonic {posargs:} +[testenv:py27-threadedresolver] +basepython = python2.7 +deps = + futures + mock + pycurl + twisted +commands = python -m tornado.test.runtests --resolver=tornado.netutil.ThreadedResolver + [testenv:pypy-full] # This configuration works with pypy 1.9. pycurl installs ok but # curl_httpclient doesn't work. Twisted works most of the time, but