From: Ben Darnell Date: Mon, 14 Sep 2015 03:13:43 +0000 (-0400) Subject: Support coroutines compiled with cython X-Git-Tag: v4.3.0b1~48 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=3463036d750bcccf0977321f73c9a07cecfe2e7e;p=thirdparty%2Ftornado.git Support coroutines compiled with cython On Python 3.5, this means supporting awaitables that are not iterables. On older versions of python, this includes * In `@gen.coroutine`, recognizing cython's coroutines via the backports_abc module. * At various points in the gen module, recognize cython's use of `StopIteration.args[0]` in place of `StopIteration.value`. * Implementing Future.__await__ and _wrap_awaitable for pre-3.3 versions of python. --- diff --git a/maint/test/cython/.gitignore b/maint/test/cython/.gitignore new file mode 100644 index 000000000..73e2ed0ce --- /dev/null +++ b/maint/test/cython/.gitignore @@ -0,0 +1,3 @@ +.eggs +cythonapp.egg-info +dist diff --git a/maint/test/cython/MANIFEST.in b/maint/test/cython/MANIFEST.in new file mode 100644 index 000000000..a42b80b11 --- /dev/null +++ b/maint/test/cython/MANIFEST.in @@ -0,0 +1 @@ +include cythonapp.pyx diff --git a/maint/test/cython/cythonapp.pyx b/maint/test/cython/cythonapp.pyx new file mode 100644 index 000000000..3bd1bc9de --- /dev/null +++ b/maint/test/cython/cythonapp.pyx @@ -0,0 +1,15 @@ +from tornado import gen +import pythonmodule + +async def native_coroutine(): + x = await pythonmodule.hello() + if x != "hello": + raise ValueError("expected hello, got %r" % x) + return "goodbye" + +@gen.coroutine +def decorated_coroutine(): + x = yield pythonmodule.hello() + if x != "hello": + raise ValueError("expected hello, got %r" % x) + return "goodbye" diff --git a/maint/test/cython/cythonapp_test.py b/maint/test/cython/cythonapp_test.py new file mode 100644 index 000000000..76427f422 --- /dev/null +++ b/maint/test/cython/cythonapp_test.py @@ -0,0 +1,22 @@ +try: + import backports_abc +except ImportError: + raise +else: + backports_abc.patch() + +from tornado.testing import AsyncTestCase, gen_test + +import cythonapp + + +class CythonCoroutineTest(AsyncTestCase): + @gen_test + def test_native_coroutine(self): + x = yield cythonapp.native_coroutine() + self.assertEqual(x, "goodbye") + + @gen_test + def test_decorated_coroutine(self): + x = yield cythonapp.decorated_coroutine() + self.assertEqual(x, "goodbye") diff --git a/maint/test/cython/pythonmodule.py b/maint/test/cython/pythonmodule.py new file mode 100644 index 000000000..e532693d1 --- /dev/null +++ b/maint/test/cython/pythonmodule.py @@ -0,0 +1,6 @@ +from tornado import gen + +@gen.coroutine +def hello(): + yield gen.sleep(0.001) + raise gen.Return("hello") diff --git a/maint/test/cython/setup.py b/maint/test/cython/setup.py new file mode 100644 index 000000000..628bb4141 --- /dev/null +++ b/maint/test/cython/setup.py @@ -0,0 +1,18 @@ +from setuptools import setup + +try: + import Cython.Build +except: + Cython = None + +if Cython is None: + ext_modules = None +else: + ext_modules=Cython.Build.cythonize('cythonapp.pyx') + +setup( + name='cythonapp', + py_modules=['cythonapp_test', 'pythonmodule'], + ext_modules=ext_modules, + setup_requires='Cython>=0.23.1', +) diff --git a/maint/test/cython/tox.ini b/maint/test/cython/tox.ini new file mode 100644 index 000000000..38949d52f --- /dev/null +++ b/maint/test/cython/tox.ini @@ -0,0 +1,19 @@ +[tox] +# This currently segfaults on pypy. +envlist = py27,py32,py33,py34,py35 + +[testenv] +deps = + ../../.. + Cython>= 0.23.1 + backports_abc + singledispatch +commands = python -m unittest cythonapp_test +# Most of these are defaults, but if you specify any you can't fall back +# defaults for the others. +basepython = + py27: python2.7 + py32: python3.2 + py33: python3.3 + py34: python3.4 + py35: python3.5 diff --git a/tornado/concurrent.py b/tornado/concurrent.py index 7a83b5079..d3d878fec 100644 --- a/tornado/concurrent.py +++ b/tornado/concurrent.py @@ -177,6 +177,16 @@ class Future(object): def __await__(self): return (yield self) """)) + else: + # Py2-compatible version for use with cython. + # Late import of gen.Return to avoid cycles. + def __await__(self): + result = yield self + # StopIteration doesn't take args before py33, + # but Cython recognizes the args tuple. + e = StopIteration() + e.args = (result,) + raise e def cancel(self): """Cancel the operation, if possible. diff --git a/tornado/gen.py b/tornado/gen.py index 5c7dc019c..34c3c2e07 100644 --- a/tornado/gen.py +++ b/tornado/gen.py @@ -101,7 +101,10 @@ except ImportError as e: try: from collections.abc import Generator as GeneratorType # py35+ except ImportError: - from types import GeneratorType + try: + from collections import Generator as GeneratorType # py2 with backports_abc + except ImportError: + from types import GeneratorType try: from inspect import isawaitable # py35+ @@ -139,6 +142,21 @@ class TimeoutError(Exception): """Exception raised by ``with_timeout``.""" +def _value_from_stopiteration(e): + try: + # StopIteration has a value attribute beginning in py33. + # So does our Return class. + return e.value + except AttributeError: + pass + try: + # Cython backports coroutine functionality by putting the value in + # e.args[0]. + return e.args[0] + except (AttributeError, IndexError): + return None + + def engine(func): """Callback-oriented decorator for asynchronous generators. @@ -236,7 +254,7 @@ def _make_coroutine_wrapper(func, replace_callback): try: result = func(*args, **kwargs) except (Return, StopIteration) as e: - result = getattr(e, 'value', None) + result = _value_from_stopiteration(e) except Exception: future.set_exc_info(sys.exc_info()) return future @@ -257,7 +275,7 @@ def _make_coroutine_wrapper(func, replace_callback): 'stack_context inconsistency (probably caused ' 'by yield within a "with StackContext" block)')) except (StopIteration, Return) as e: - future.set_result(getattr(e, 'value', None)) + future.set_result(_value_from_stopiteration(e)) except Exception: future.set_exc_info(sys.exc_info()) else: @@ -302,6 +320,8 @@ class Return(Exception): def __init__(self, value=None): super(Return, self).__init__() self.value = value + # Cython recognizes subclasses of StopIteration with a .args tuple. + self.args = (value,) class WaitIterator(object): @@ -946,7 +966,7 @@ class Runner(object): raise LeakedCallbackError( "finished without waiting for callbacks %r" % self.pending_callbacks) - self.result_future.set_result(getattr(e, 'value', None)) + self.result_future.set_result(_value_from_stopiteration(e)) self.result_future = None self._deactivate_stack_context() return @@ -1057,11 +1077,57 @@ if sys.version_info >= (3, 3): exec(textwrap.dedent(""" @coroutine def _wrap_awaitable(x): + if hasattr(x, '__await__'): + x = x.__await__() return (yield from x) """)) else: + # Py2-compatible version for use with Cython. + # Copied from PEP 380. + @coroutine def _wrap_awaitable(x): - raise NotImplementedError() + if hasattr(x, '__await__'): + _i = x.__await__() + else: + _i = iter(x) + try: + _y = next(_i) + except StopIteration as _e: + _r = _value_from_stopiteration(_e) + else: + while 1: + try: + _s = yield _y + except GeneratorExit as _e: + try: + _m = _i.close + except AttributeError: + pass + else: + _m() + raise _e + except BaseException as _e: + _x = sys.exc_info() + try: + _m = _i.throw + except AttributeError: + raise _e + else: + try: + _y = _m(*_x) + except StopIteration as _e: + _r = _value_from_stopiteration(_e) + break + else: + try: + if _s is None: + _y = next(_i) + else: + _y = _i.send(_s) + except StopIteration as _e: + _r = _value_from_stopiteration(_e) + break + raise Return(_r) def convert_yielded(yielded):