From: Ben Darnell Date: Sun, 7 Oct 2012 21:21:42 +0000 (-0700) Subject: Implement TwistedIOLoop, to bridge the gap with Twisted in the other direction. X-Git-Tag: v3.0.0~237 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=3654790fcb687c8487f6fc40cfac863734ded5e6;p=thirdparty%2Ftornado.git Implement TwistedIOLoop, to bridge the gap with Twisted in the other direction. This also serves as a proof of concept for the refactored IOLoop interface. --- diff --git a/tornado/platform/twisted.py b/tornado/platform/twisted.py index 1ce0e6bb8..ff43e6334 100644 --- a/tornado/platform/twisted.py +++ b/tornado/platform/twisted.py @@ -16,19 +16,25 @@ # Note: This module's docs are not currently extracted automatically, # so changes must be made manually to twisted.rst # TODO: refactor doc build process to use an appropriate virtualenv -"""A Twisted reactor built on the Tornado IOLoop. +"""Bridges between the Twisted reactor and Tornado IOLoop. This module lets you run applications and libraries written for -Twisted in a Tornado application. To use it, simply call `install` at -the beginning of the application:: +Twisted in a Tornado application. It can be used in two modes, +depending on which library's underlying event loop you want to use. + +Twisted on Tornado +------------------ + +`TornadoReactor` implements the Twisted reactor interface on top of +the Tornado IOLoop. To use it, simply call `install` at the beginning +of the application:: import tornado.platform.twisted tornado.platform.twisted.install() from twisted.internet import reactor When the app is ready to start, call `IOLoop.instance().start()` -instead of `reactor.run()`. This will allow you to use a mixture of -Twisted and Tornado code in the same process. +instead of `reactor.run()`. It is also possible to create a non-global reactor by calling `tornado.platform.twisted.TornadoReactor(io_loop)`. However, if @@ -41,17 +47,32 @@ recommended to call:: before closing the `IOLoop`. -This module has been tested with Twisted versions 11.0.0, 11.1.0, and 12.0.0 +Tornado on Twisted +------------------ + +`TwistedIOLoop` implements the Tornado IOLoop interface on top of the Twisted +reactor. Recommended usage:: + + from tornado.platform.twisted import TwistedIOLoop + from twisted.internet import reactor + TwistedIOLoop().install() + # Set up your tornado application as usual using `IOLoop.instance` + reactor.run() + +`TwistedIOLoop` always uses the global Twisted reactor. + +This module has been tested with Twisted versions 11.0.0 and newer. """ from __future__ import absolute_import, division, with_statement import functools +import datetime import time from twisted.internet.posixbase import PosixReactorBase from twisted.internet.interfaces import \ - IReactorFDSet, IDelayedCall, IReactorTime + IReactorFDSet, IDelayedCall, IReactorTime, IReadDescriptor, IWriteDescriptor from twisted.python import failure, log from twisted.internet import error @@ -60,7 +81,7 @@ from zope.interface import implementer import tornado import tornado.ioloop from tornado.log import app_log -from tornado.stack_context import NullContext +from tornado.stack_context import NullContext, wrap from tornado.ioloop import IOLoop @@ -328,3 +349,107 @@ def install(io_loop=None): from twisted.internet.main import installReactor installReactor(reactor) return reactor + +class _FD(object): + def __init__(self, fd, handler): + self.fd = fd + self.handler = handler + self.reading = False + self.writing = False + + def fileno(self): + return self.fd + + def doRead(self): + self.handler(self.fd, tornado.ioloop.IOLoop.READ) + + def doWrite(self): + self.handler(self.fd, tornado.ioloop.IOLoop.WRITE) + + def connectionLost(self, reason): + self.handler(self.fd, tornado.ioloop.IOLoop.ERROR) + + def logPrefix(self): + return '' +_FD = implementer(IReadDescriptor, IWriteDescriptor)(_FD) + +class TwistedIOLoop(tornado.ioloop.IOLoop): + """IOLoop implementation that runs on Twisted. + + Uses the global Twisted reactor. It is possible to create multiple + TwistedIOLoops in the same process, but it doesn't really make sense + because they will all run in the same thread. + + Not compatible with `tornado.process.Subprocess.set_exit_callback` + because the ``SIGCHLD`` handlers used by Tornado and Twisted conflict + with each other. + """ + def initialize(self): + from twisted.internet import reactor + self.reactor = reactor + self.fds = {} + + def close(self, all_fds=False): + self.reactor.removeAll() + for c in self.reactor.getDelayedCalls(): + c.cancel() + + def add_handler(self, fd, handler, events): + if fd in self.fds: + raise ValueError('fd %d added twice' % fd) + self.fds[fd] = _FD(fd, wrap(handler)) + if events | tornado.ioloop.IOLoop.READ: + self.fds[fd].reading = True + self.reactor.addReader(self.fds[fd]) + if events | tornado.ioloop.IOLoop.WRITE: + self.fds[fd].writing = True + self.reactor.addWriter(self.fds[fd]) + + def update_handler(self, fd, events): + if events | tornado.ioloop.IOLoop.READ: + if not self.fds[fd].reading: + self.fds[fd].reading = True + self.reactor.addReader(self.fds[fd]) + else: + if self.fds[fd].reading: + self.fds[fd].reading = False + self.reactor.removeReader(self.fds[fd]) + if events | tornado.ioloop.IOLoop.WRITE: + if not self.fds[fd].writing: + self.fds[fd].writing = True + self.reactor.addWriter(self.fds[fd]) + else: + if self.fds[fd].writing: + self.fds[fd].writing = False + self.reactor.removeWriter(self.fds[fd]) + + def remove_handler(self, fd): + if self.fds[fd].reading: + self.reactor.removeReader(self.fds[fd]) + if self.fds[fd].writing: + self.reactor.removeWriter(self.fds[fd]) + del self.fds[fd] + + def start(self): + self.reactor.run() + + def stop(self): + self.reactor.crash() + + def add_timeout(self, deadline, callback): + if isinstance(deadline, (int, long, float)): + delay = max(deadline - self.time(), 0) + elif isinstance(deadline, datetime.timedelta): + delay = deadline.total_seconds() + else: + raise TypeError("Unsupported deadline %r") + return self.reactor.callLater(delay, wrap(callback)) + + def remove_timeout(self, timeout): + timeout.cancel() + + def add_callback(self, callback): + self.reactor.callFromThread(wrap(callback)) + + def add_callback_from_signal(self, callback): + self.add_callback(callback) diff --git a/tornado/test/process_test.py b/tornado/test/process_test.py index bfb4b4c0c..edd7f2b49 100644 --- a/tornado/test/process_test.py +++ b/tornado/test/process_test.py @@ -144,7 +144,12 @@ class SubprocessTest(AsyncTestCase): data = self.wait() self.assertEqual(data, b("")) + def skip_if_twisted(self): + if self.io_loop.__class__.__name__ == 'TwistedIOLoop': + raise unittest.SkipTest("SIGCHLD not compatible with Twisted IOLoop") + def test_sigchild(self): + self.skip_if_twisted() Subprocess.initialize(io_loop=self.io_loop) self.addCleanup(Subprocess.uninitialize) subproc = Subprocess([sys.executable, '-c', 'pass'], @@ -155,6 +160,7 @@ class SubprocessTest(AsyncTestCase): self.assertEqual(subproc.returncode, ret) def test_sigchild_signal(self): + self.skip_if_twisted() Subprocess.initialize(io_loop=self.io_loop) self.addCleanup(Subprocess.uninitialize) subproc = Subprocess([sys.executable, '-c', diff --git a/tornado/test/twisted_test.py b/tornado/test/twisted_test.py index 7521d7406..254433670 100644 --- a/tornado/test/twisted_test.py +++ b/tornado/test/twisted_test.py @@ -35,7 +35,7 @@ try: from twisted.web.resource import Resource from twisted.web.server import Site from twisted.python import log - from tornado.platform.twisted import TornadoReactor + from tornado.platform.twisted import TornadoReactor, TwistedIOLoop from zope.interface import implementer have_twisted = True except ImportError: @@ -58,7 +58,11 @@ def save_signal_handlers(): for sig in [signal.SIGINT, signal.SIGTERM, signal.SIGCHLD]: saved[sig] = signal.getsignal(sig) if "twisted" in repr(saved): - raise Exception("twisted signal handlers already installed") + if not issubclass(IOLoop.configured_class(), TwistedIOLoop): + # when the global ioloop is twisted, we expect the signal + # handlers to be installed. Otherwise, it means we're not + # cleaning up after twisted properly. + raise Exception("twisted signal handlers already installed") return saved @@ -448,6 +452,9 @@ if have_twisted: twisted_tests = { 'twisted.internet.test.test_core.ObjectModelIntegrationTest': [], 'twisted.internet.test.test_core.SystemEventTestsBuilder': [ + 'test_iterate', # deliberately not supported + 'test_runAfterCrash', # fails because TwistedIOLoop uses the global reactor + ] if issubclass(IOLoop.configured_class(), TwistedIOLoop) else [ 'test_iterate', # deliberately not supported ], 'twisted.internet.test.test_fdset.ReactorFDSetTestsBuilder': [ diff --git a/tornado/util.py b/tornado/util.py index 967470839..f550449a0 100644 --- a/tornado/util.py +++ b/tornado/util.py @@ -122,9 +122,7 @@ class Configurable(object): base = cls.configurable_base() args = {} if cls is base: - if cls.__impl_class is None: - base.__impl_class = cls.configurable_default() - impl = base.__impl_class + impl = cls.configured_class() if base.__impl_kwargs: args.update(base.__impl_kwargs) else: @@ -173,6 +171,15 @@ class Configurable(object): base.__impl_class = impl base.__impl_kwargs = kwargs + @classmethod + def configured_class(cls): + """Returns the currently configured class.""" + base = cls.configurable_base() + if cls.__impl_class is None: + base.__impl_class = cls.configurable_default() + return base.__impl_class + + @classmethod def _save_configuration(cls): base = cls.configurable_base() diff --git a/tox.ini b/tox.ini index 9adf8b534..4367b5725 100644 --- a/tox.ini +++ b/tox.ini @@ -11,7 +11,7 @@ [tox] # "-full" variants include optional dependencies, to ensure # that things work both in a bare install and with all the extras. -envlist = py27-full, py27-curl, py25-full, py32, pypy, py25, py26, py26-full, py27, py32-utf8, py33, py27-opt, py32-opt, pypy-full, py27-select, py27-monotonic, py33-monotonic +envlist = py27-full, py27-curl, py25-full, py32, pypy, py25, py26, py26-full, py27, py32-utf8, py33, py27-opt, py32-opt, pypy-full, py27-select, py27-monotonic, py33-monotonic, py27-twisted [testenv] commands = python -m tornado.test.runtests {posargs:} @@ -85,6 +85,14 @@ deps = twisted>=12.0.0 commands = python -m tornado.test.runtests --ioloop=tornado.platform.select.SelectIOLoop {posargs:} +[testenv:py27-twisted] +basepython = python2.7 +deps = + futures + pycurl + twisted>=12.2.0 +commands = python -m tornado.test.runtests --ioloop=tornado.platform.twisted.TwistedIOLoop {posargs:} + [testenv:py27-monotonic] basepython = python2.7 # TODO: remove this url when the pypi page is updated. diff --git a/website/sphinx/releases/next.rst b/website/sphinx/releases/next.rst index 7edd47171..e5a2b0a7d 100644 --- a/website/sphinx/releases/next.rst +++ b/website/sphinx/releases/next.rst @@ -146,3 +146,7 @@ In progress creating multiple periodic callbacks). Starting autoreload on more than one `IOLoop` in the same process now logs a warning. * Method `IOLoop.running()` has been removed. +* `IOLoop` has been refactored to better support subclassing. +* New class `tornado.platform.twisted.TwistedIOLoop` allows Tornado + code to be run on the Twisted reactor (as opposed to the existing + `TornadoReactor`, which bridges the gap in the other direction). diff --git a/website/sphinx/twisted.rst b/website/sphinx/twisted.rst index 6a4d85416..2e5222dfb 100644 --- a/website/sphinx/twisted.rst +++ b/website/sphinx/twisted.rst @@ -1,20 +1,25 @@ -``tornado.platform.twisted`` --- Run code written for Twisted on Tornado +``tornado.platform.twisted`` --- Bridges between Twisted and Tornado ======================================================================== .. module:: tornado.platform.twisted -This module contains a Twisted reactor build on the Tornado IOLoop, -which lets you run applications and libraries written for Twisted in a -Tornado application. To use it, simply call `install` at the -beginning of the application:: +This module lets you run applications and libraries written for +Twisted in a Tornado application. It can be used in two modes, +depending on which library's underlying event loop you want to use. + +Twisted on Tornado +------------------ + +`TornadoReactor` implements the Twisted reactor interface on top of +the Tornado IOLoop. To use it, simply call `install` at the beginning +of the application:: import tornado.platform.twisted tornado.platform.twisted.install() from twisted.internet import reactor When the app is ready to start, call `IOLoop.instance().start()` -instead of `reactor.run()`. This will allow you to use a mixture of -Twisted and Tornado code in the same process. +instead of `reactor.run()`. It is also possible to create a non-global reactor by calling `tornado.platform.twisted.TornadoReactor(io_loop)`. However, if @@ -27,19 +32,18 @@ recommended to call:: before closing the `IOLoop`. -This module has been tested with Twisted versions 11.0.0 and 11.1.0. +Tornado on Twisted +------------------ -.. function:: install(io_loop=None) +`TwistedIOLoop` implements the Tornado IOLoop interface on top of the Twisted +reactor. Recommended usage:: -Install this package as the default Twisted reactor. - -.. class:: TornadoReactor(io_loop=None) + from tornado.platform.twisted import TwistedIOLoop + from twisted.internet import reactor + TwistedIOLoop().install() + # Set up your tornado application as usual using `IOLoop.instance` + reactor.run() -Twisted reactor built on the Tornado IOLoop. +`TwistedIOLoop` always uses the global Twisted reactor. -Since it is intented to be used in applications where the top-level -event loop is ``io_loop.start()`` rather than ``reactor.run()``, -it is implemented a little differently than other Twisted reactors. -We override `mainLoop` instead of `doIteration` and must implement -timed call functionality on top of `IOLoop.add_timeout` rather than -using the implementation in `PosixReactorBase`. +This module has been tested with Twisted versions 11.0.0 and newer.