Make Resolver Configurable and test with a threaded resolver.

diff --git a/tornado/netutil.py b/tornado/netutil.py
index 05d8468ba17d5a2fda19a057998ffb33c8cbb8ba..4003245e3b93052017cdc251093fd0174be35592 100644 (file)
--- a/tornado/netutil.py
+++ b/tornado/netutil.py
@@ -28,6 +28,7 @@ import stat
 from tornado.concurrent import dummy_executor, run_on_executor
 from tornado.ioloop import IOLoop
 from tornado.platform.auto import set_close_exec
+from tornado.util import Configurable
 
 
 def bind_sockets(port, address=None, family=socket.AF_UNSPEC, backlog=128, flags=None):
@@ -141,7 +142,15 @@ def add_accept_handler(sock, callback, io_loop=None):
     io_loop.add_handler(sock.fileno(), accept_handler, IOLoop.READ)
 
 
-class BaseResolver(object):
+class Resolver(Configurable):
+    @classmethod
+    def configurable_base(cls):
+        return Resolver
+
+    @classmethod
+    def configurable_default(cls):
+        return BlockingResolver
+
     def getaddrinfo(self, *args, **kwargs):
         """Resolves an address.
 
@@ -157,8 +166,8 @@ class BaseResolver(object):
         raise NotImplementedError()
 
 
-class Resolver(BaseResolver):
-    def __init__(self, io_loop=None, executor=None):
+class ExecutorResolver(Resolver):
+    def initialize(self, io_loop=None, executor=None):
         self.io_loop = io_loop or IOLoop.instance()
         self.executor = executor or dummy_executor
 
@@ -166,7 +175,17 @@ class Resolver(BaseResolver):
     def getaddrinfo(self, *args, **kwargs):
         return socket.getaddrinfo(*args, **kwargs)
 
-class OverrideResolver(BaseResolver):
+class BlockingResolver(ExecutorResolver):
+    def initialize(self, io_loop=None):
+        super(BlockingResolver, self).initialize(io_loop=io_loop)
+
+class ThreadedResolver(ExecutorResolver):
+    def initialize(self, io_loop=None, num_threads=10):
+        from concurrent.futures import ThreadPoolExecutor
+        super(ThreadedResolver, self).initialize(
+            io_loop=io_loop, executor=ThreadPoolExecutor(num_threads))
+
+class OverrideResolver(Resolver):
     """Wraps a resolver with a mapping of overrides.
 
     This can be used to make local DNS changes (e.g. for testing)
@@ -174,7 +193,7 @@ class OverrideResolver(BaseResolver):
 
     The mapping can contain either host strings or host-port pairs.
     """
-    def __init__(self, resolver, mapping):
+    def initialize(self, resolver, mapping):
         self.resolver = resolver
         self.mapping = mapping
 

diff --git a/tornado/simple_httpclient.py b/tornado/simple_httpclient.py
index 73739ff4002b44a01b987778b075c68a62eeb574..d659318daf8c5a51b5091c2175b24d11a623b823 100644 (file)
--- a/tornado/simple_httpclient.py
+++ b/tornado/simple_httpclient.py
@@ -75,7 +75,8 @@ class SimpleAsyncHTTPClient(AsyncHTTPClient):
         self.max_buffer_size = max_buffer_size
         self.resolver = resolver or Resolver(io_loop=io_loop)
         if hostname_mapping is not None:
-            self.resolver = OverrideResolver(self.resolver, hostname_mapping)
+            self.resolver = OverrideResolver(resolver=self.resolver,
+                                             mapping=hostname_mapping)
 
     def fetch_impl(self, request, callback):
         self.queue.append((request, callback))

diff --git a/tornado/test/httpserver_test.py b/tornado/test/httpserver_test.py
index 87a3e3e83d850f238e332cf0c61efcfc6afa0e8a..2a60e9de1871f0b4e0b737a65825d8d9282efaca 100644 (file)
--- a/tornado/test/httpserver_test.py
+++ b/tornado/test/httpserver_test.py
@@ -190,7 +190,7 @@ class HTTPConnectionTest(AsyncHTTPTestCase):
                 httpclient.HTTPRequest(self.get_url("/")),
                 dict(httpclient.HTTPRequest._DEFAULTS)),
             None, self.stop,
-            1024 * 1024, Resolver(self.io_loop))
+            1024 * 1024, Resolver(io_loop=self.io_loop))
         conn.set_request(
             b"\r\n".join(headers +
                          [utf8("Content-Length: %d\r\n" % len(body))]) +

diff --git a/tornado/test/netutil_test.py b/tornado/test/netutil_test.py
index 4342e18e319d1d69fa04c45524396d2d6e36423a..4711019fae311bef5231194a4689d37fa3be5d74 100644 (file)
--- a/tornado/test/netutil_test.py
+++ b/tornado/test/netutil_test.py
@@ -2,7 +2,7 @@ from __future__ import absolute_import, division, print_function, with_statement
 
 import socket
 
-from tornado.netutil import Resolver
+from tornado.netutil import BlockingResolver, ThreadedResolver
 from tornado.testing import AsyncTestCase
 from tornado.test.util import unittest
 
@@ -28,20 +28,18 @@ class _ResolverTestMixin(object):
             future.result())
 
 
-class SyncResolverTest(AsyncTestCase, _ResolverTestMixin):
+class BlockingResolverTest(AsyncTestCase, _ResolverTestMixin):
     def setUp(self):
-        super(SyncResolverTest, self).setUp()
-        self.resolver = Resolver(self.io_loop)
+        super(BlockingResolverTest, self).setUp()
+        self.resolver = BlockingResolver(io_loop=self.io_loop)
 
 
 @unittest.skipIf(futures is None, "futures module not present")
 class ThreadedResolverTest(AsyncTestCase, _ResolverTestMixin):
     def setUp(self):
         super(ThreadedResolverTest, self).setUp()
-        from concurrent.futures import ThreadPoolExecutor
-        self.executor = ThreadPoolExecutor(2)
-        self.resolver = Resolver(self.io_loop, self.executor)
+        self.resolver = ThreadedResolver(io_loop=self.io_loop)
 
     def tearDown(self):
-        self.executor.shutdown()
+        self.resolver.executor.shutdown()
         super(ThreadedResolverTest, self).tearDown()

diff --git a/tornado/test/runtests.py b/tornado/test/runtests.py
index 24717114eec7331500b066b5476627829b43052d..e9e21bda840212662ecb30bfdaff870916fabf63 100644 (file)
--- a/tornado/test/runtests.py
+++ b/tornado/test/runtests.py
@@ -6,6 +6,7 @@ import textwrap
 import sys
 from tornado.httpclient import AsyncHTTPClient
 from tornado.ioloop import IOLoop
+from tornado.netutil import Resolver
 from tornado.options import define, options, add_parse_callback
 from tornado.test.util import unittest
 
@@ -85,6 +86,8 @@ if __name__ == '__main__':
            callback=AsyncHTTPClient.configure)
     define('ioloop', type=str, default=None)
     define('ioloop_time_monotonic', default=False)
+    define('resolver', type=str, default=None,
+           callback=Resolver.configure)
 
     def configure_ioloop():
         kwargs = {}

diff --git a/tornado/websocket.py b/tornado/websocket.py
index 3c914d99b242b75a5f3d0dc9c2f06e74d15e27b8..235a1102e6356f6cf72b825ecdf871a39c31fa99 100644 (file)
--- a/tornado/websocket.py
+++ b/tornado/websocket.py
@@ -746,7 +746,7 @@ class _WebSocketClientConnection(simple_httpclient._HTTPConnection):
 
         super(_WebSocketClientConnection, self).__init__(
             io_loop, None, request, lambda: None, lambda response: None,
-            104857600, Resolver(io_loop))
+            104857600, Resolver(io_loop=io_loop))
 
     def _on_close(self):
         self.on_message(None)

diff --git a/tox.ini b/tox.ini
index cfbf468df61a9e41527957d4fc2f2a9bc22322e4..bc0f73a674ce9454000c804221b80511c043a6e7 100644 (file)
--- a/tox.ini
+++ b/tox.ini
@@ -11,7 +11,7 @@
 [tox]
 # "-full" variants include optional dependencies, to ensure
 # that things work both in a bare install and with all the extras.
-envlist = py27-full, py27-curl, py32-full, pypy, py26, py26-full, py27, py32, py32-utf8, py33, py27-opt, py32-opt, pypy-full, py27-select, py27-monotonic, py33-monotonic, py27-twisted
+envlist = py27-full, py27-curl, py32-full, pypy, py26, py26-full, py27, py32, py32-utf8, py33, py27-opt, py32-opt, pypy-full, py27-select, py27-monotonic, py33-monotonic, py27-twisted, py27-threadedresolver
 [testenv]
 commands = python -m tornado.test.runtests {posargs:}
 
@@ -91,6 +91,15 @@ deps =
      twisted
 commands = python -m tornado.test.runtests --ioloop_time_monotonic {posargs:}
 
+[testenv:py27-threadedresolver]
+basepython = python2.7
+deps =
+     futures
+     mock
+     pycurl
+     twisted
+commands = python -m tornado.test.runtests --resolver=tornado.netutil.ThreadedResolver
+
 [testenv:pypy-full]
 # This configuration works with pypy 1.9.  pycurl installs ok but
 # curl_httpclient doesn't work.  Twisted works most of the time, but