From: Ben Darnell Date: Sat, 23 Dec 2017 02:14:15 +0000 (-0500) Subject: netutil: Use a threaded resolver by default X-Git-Tag: v5.0.0~30^2~1 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=1d113d861e09c6f2996039cb8b21253ba4bad2eb;p=thirdparty%2Ftornado.git netutil: Use a threaded resolver by default Uses IOLoop.run_in_executor. concurrent.futures is now required on Python 2. --- diff --git a/.travis.yml b/.travis.yml index cd2021bd6..df37efa50 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,8 +20,8 @@ python: - pypy3.5-5.8.0 install: - - if [[ $TRAVIS_PYTHON_VERSION == 2* ]]; then travis_retry pip install futures mock monotonic; fi - - if [[ $TRAVIS_PYTHON_VERSION == 'pypy' ]]; then travis_retry pip install futures mock; fi + - if [[ $TRAVIS_PYTHON_VERSION == 2* ]]; then travis_retry pip install mock monotonic; fi + - if [[ $TRAVIS_PYTHON_VERSION == 'pypy' ]]; then travis_retry pip install mock; fi # TODO(bdarnell): pycares tests are currently disabled on travis due to ipv6 issues. #- if [[ $TRAVIS_PYTHON_VERSION != 'pypy'* ]]; then travis_retry pip install pycares; fi - if [[ $TRAVIS_PYTHON_VERSION != 'pypy'* ]]; then travis_retry pip install pycurl; fi @@ -66,7 +66,6 @@ script: - LANG=C python $TARGET - LANG=en_US.utf-8 python $TARGET - if [[ $TRAVIS_PYTHON_VERSION == 3* ]]; then python -bb $TARGET; fi - - if [[ $TRAVIS_PYTHON_VERSION != pypy* ]]; then python $TARGET --resolver=tornado.netutil.ThreadedResolver; fi - if [[ $TRAVIS_PYTHON_VERSION == 2* ]]; then python $TARGET --httpclient=tornado.curl_httpclient.CurlAsyncHTTPClient; fi - if [[ $TRAVIS_PYTHON_VERSION == 2* ]]; then python $TARGET --ioloop=tornado.platform.twisted.TwistedIOLoop; fi - if [[ $TRAVIS_PYTHON_VERSION == 2* ]]; then python $TARGET --resolver=tornado.platform.twisted.TwistedResolver; fi diff --git a/docs/guide/async.rst b/docs/guide/async.rst index 5082f3fba..60f8a23b3 100644 --- a/docs/guide/async.rst +++ b/docs/guide/async.rst @@ -27,11 +27,7 @@ hundreds of milliseconds of CPU time, far more than a typical network or disk access). A function can be blocking in some respects and non-blocking in -others. For example, `tornado.httpclient` in the default -configuration blocks on DNS resolution but not on other network access -(to mitigate this use `.ThreadedResolver` or a -``tornado.curl_httpclient`` with a properly-configured build of -``libcurl``). In the context of Tornado we generally talk about +others. In the context of Tornado we generally talk about blocking in the context of network I/O, although all kinds of blocking are to be minimized. diff --git a/docs/index.rst b/docs/index.rst index ed24eb623..ce4156c4f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -73,10 +73,6 @@ older python versions). In addition to the requirements which will be installed automatically by ``pip`` or ``setup.py install``, the following optional packages may be useful: -* `concurrent.futures `_ is the - recommended thread pool for use with Tornado and enables the use of - `~tornado.netutil.ThreadedResolver`. It is needed only on Python 2; - Python 3 includes this package in the standard library. * `pycurl `_ is used by the optional ``tornado.curl_httpclient``. Libcurl version 7.22 or higher is required. * `Twisted `_ may be used with the classes in diff --git a/docs/releases/v5.0.0.rst b/docs/releases/v5.0.0.rst index 6456090d8..0ef227a98 100644 --- a/docs/releases/v5.0.0.rst +++ b/docs/releases/v5.0.0.rst @@ -71,6 +71,8 @@ Backwards-compatibility notes Other notes ~~~~~~~~~~~ +- The ``futures`` (`concurrent.futures` backport) package is now required + on Python 2.7. - The ``certifi`` and ``backports.ssl-match-hostname`` packages are no longer required on Python 2.7. - Python 3.6 or higher is recommended, because it features more @@ -204,6 +206,9 @@ Other notes `tornado.netutil` ~~~~~~~~~~~~~~~~~ +- The default `.Resolver` now uses `.IOLoop.run_in_executor`. + `.ExecutorResolver`, `.BlockingResolver`, and `.ThreadedResolver` are + deprecated. - The ``io_loop`` arguments to `.add_accept_handler`, `.ExecutorResolver`, and `.ThreadedResolver` have been removed. - `.add_accept_handler` returns a callable which can be used to remove diff --git a/setup.py b/setup.py index 26bb716d5..37ea05942 100644 --- a/setup.py +++ b/setup.py @@ -127,6 +127,8 @@ if (platform.python_implementation() == 'CPython' and if setuptools is not None: # If setuptools is not available, you're on your own for dependencies. install_requires = [] + if sys.version_info < (3, 2): + install_requires.append('futures') if sys.version_info < (3, 4): install_requires.append('singledispatch') if sys.version_info < (3, 5): diff --git a/tornado/netutil.py b/tornado/netutil.py index 1469b90c0..caaa09090 100644 --- a/tornado/netutil.py +++ b/tornado/netutil.py @@ -25,6 +25,7 @@ import socket import stat from tornado.concurrent import dummy_executor, run_on_executor +from tornado import gen from tornado.ioloop import IOLoop from tornado.platform.auto import set_close_exec from tornado.util import PY3, Configurable, errno_from_exception @@ -292,11 +293,16 @@ class Resolver(Configurable): The implementations of this interface included with Tornado are - * `tornado.netutil.BlockingResolver` - * `tornado.netutil.ThreadedResolver` + * `tornado.netutil.DefaultExecutorResolver` + * `tornado.netutil.BlockingResolver` (deprecated) + * `tornado.netutil.ThreadedResolver` (deprecated) * `tornado.netutil.OverrideResolver` * `tornado.platform.twisted.TwistedResolver` * `tornado.platform.caresresolver.CaresResolver` + + .. versionchanged:: 5.0 + The default implementation has changed from `BlockingResolver` to + `DefaultExecutorResolver`. """ @classmethod def configurable_base(cls): @@ -304,7 +310,7 @@ class Resolver(Configurable): @classmethod def configurable_default(cls): - return BlockingResolver + return DefaultExecutorResolver def resolve(self, host, port, family=socket.AF_UNSPEC, callback=None): """Resolves an address. @@ -335,6 +341,31 @@ class Resolver(Configurable): pass +def _resolve_addr(host, port, family=socket.AF_UNSPEC): + # On Solaris, getaddrinfo fails if the given port is not found + # in /etc/services and no socket type is given, so we must pass + # one here. The socket type used here doesn't seem to actually + # matter (we discard the one we get back in the results), + # so the addresses we return should still be usable with SOCK_DGRAM. + addrinfo = socket.getaddrinfo(host, port, family, socket.SOCK_STREAM) + results = [] + for family, socktype, proto, canonname, address in addrinfo: + results.append((family, address)) + return results + + +class DefaultExecutorResolver(Resolver): + """Resolver implementation using `.IOLoop.run_in_executor`. + + .. versionadded:: 5.0 + """ + @gen.coroutine + def resolve(self, host, port, family=socket.AF_UNSPEC): + result = yield IOLoop.current().run_in_executor( + None, _resolve_addr, host, port, family) + raise gen.Return(result) + + class ExecutorResolver(Resolver): """Resolver implementation using a `concurrent.futures.Executor`. @@ -347,6 +378,10 @@ class ExecutorResolver(Resolver): .. versionchanged:: 5.0 The ``io_loop`` argument (deprecated since version 4.1) has been removed. + + .. deprecated:: 5.0 + The default `Resolver` now uses `.IOLoop.run_in_executor`; use that instead + of this class. """ def initialize(self, executor=None, close_executor=True): self.io_loop = IOLoop.current() @@ -364,16 +399,7 @@ class ExecutorResolver(Resolver): @run_on_executor def resolve(self, host, port, family=socket.AF_UNSPEC): - # On Solaris, getaddrinfo fails if the given port is not found - # in /etc/services and no socket type is given, so we must pass - # one here. The socket type used here doesn't seem to actually - # matter (we discard the one we get back in the results), - # so the addresses we return should still be usable with SOCK_DGRAM. - addrinfo = socket.getaddrinfo(host, port, family, socket.SOCK_STREAM) - results = [] - for family, socktype, proto, canonname, address in addrinfo: - results.append((family, address)) - return results + return _resolve_addr(host, port, family) class BlockingResolver(ExecutorResolver): @@ -381,6 +407,10 @@ class BlockingResolver(ExecutorResolver): The `.IOLoop` will be blocked during the resolution, although the callback will not be run until the next `.IOLoop` iteration. + + .. deprecated:: 5.0 + The default `Resolver` now uses `.IOLoop.run_in_executor`; use that instead + of this class. """ def initialize(self): super(BlockingResolver, self).initialize() @@ -401,6 +431,10 @@ class ThreadedResolver(ExecutorResolver): .. versionchanged:: 3.1 All ``ThreadedResolvers`` share a single thread pool, whose size is set by the first one to be created. + + .. deprecated:: 5.0 + The default `Resolver` now uses `.IOLoop.run_in_executor`; use that instead + of this class. """ _threadpool = None # type: ignore _threadpool_pid = None # type: int diff --git a/tornado/platform/twisted.py b/tornado/platform/twisted.py index f8d32193e..43e447c13 100644 --- a/tornado/platform/twisted.py +++ b/tornado/platform/twisted.py @@ -520,7 +520,7 @@ class TwistedResolver(Resolver): recommended only when threads cannot be used, since it has limitations compared to the standard ``getaddrinfo``-based `~tornado.netutil.Resolver` and - `~tornado.netutil.ThreadedResolver`. Specifically, it returns at + `~tornado.netutil.DefaultExecutorResolver`. Specifically, it returns at most one result, and arguments other than ``host`` and ``family`` are ignored. It may fail to resolve when ``family`` is not ``socket.AF_UNSPEC``. diff --git a/tox.ini b/tox.ini index 2cc0a67ce..603914b54 100644 --- a/tox.ini +++ b/tox.ini @@ -32,7 +32,6 @@ envlist = py2-twistedlayered, # Alternate Resolvers. - {py2,py3}-full-{threadedresolver}, {py2,py3}-full-caresresolver, # Other configurations; see comments below. @@ -73,8 +72,6 @@ deps = {py27,py34,py35,py36}-full: twisted {py2,py3}: twisted {py2,py3,py27,py34,py35,py36}-full: pycares - # futures became standard in py32 - {py2,py27,pypy}-full: futures # mock became standard in py33 {py2,py27,pypy,pypy3}-full: mock # singledispatch became standard in py34.