Support coroutines compiled with cython

On Python 3.5, this means supporting awaitables that are not
iterables. On older versions of python, this includes
* In `@gen.coroutine`, recognizing cython's coroutines
  via the backports_abc module.
* At various points in the gen module, recognize cython's
  use of `StopIteration.args[0]` in place of `StopIteration.value`.
* Implementing Future.__await__ and _wrap_awaitable for
  pre-3.3 versions of python.

diff --git a/maint/test/cython/.gitignore b/maint/test/cython/.gitignore
new file mode 100644 (file)
index 0000000..73e2ed0
--- /dev/null
+++ b/maint/test/cython/.gitignore
@@ -0,0 +1,3 @@
+.eggs
+cythonapp.egg-info
+dist

diff --git a/maint/test/cython/MANIFEST.in b/maint/test/cython/MANIFEST.in
new file mode 100644 (file)
index 0000000..a42b80b
--- /dev/null
+++ b/maint/test/cython/MANIFEST.in
@@ -0,0 +1 @@
+include cythonapp.pyx

diff --git a/maint/test/cython/cythonapp.pyx b/maint/test/cython/cythonapp.pyx
new file mode 100644 (file)
index 0000000..3bd1bc9
--- /dev/null
+++ b/maint/test/cython/cythonapp.pyx
@@ -0,0 +1,15 @@
+from tornado import gen
+import pythonmodule
+
+async def native_coroutine():
+    x = await pythonmodule.hello()
+    if x != "hello":
+        raise ValueError("expected hello, got %r" % x)
+    return "goodbye"
+
+@gen.coroutine
+def decorated_coroutine():
+    x = yield pythonmodule.hello()
+    if x != "hello":
+        raise ValueError("expected hello, got %r" % x)
+    return "goodbye"

diff --git a/maint/test/cython/cythonapp_test.py b/maint/test/cython/cythonapp_test.py
new file mode 100644 (file)
index 0000000..76427f4
--- /dev/null
+++ b/maint/test/cython/cythonapp_test.py
@@ -0,0 +1,22 @@
+try:
+    import backports_abc
+except ImportError:
+    raise
+else:
+    backports_abc.patch()
+
+from tornado.testing import AsyncTestCase, gen_test
+
+import cythonapp
+
+
+class CythonCoroutineTest(AsyncTestCase):
+    @gen_test
+    def test_native_coroutine(self):
+        x = yield cythonapp.native_coroutine()
+        self.assertEqual(x, "goodbye")
+
+    @gen_test
+    def test_decorated_coroutine(self):
+        x = yield cythonapp.decorated_coroutine()
+        self.assertEqual(x, "goodbye")

diff --git a/maint/test/cython/pythonmodule.py b/maint/test/cython/pythonmodule.py
new file mode 100644 (file)
index 0000000..e532693
--- /dev/null
+++ b/maint/test/cython/pythonmodule.py
@@ -0,0 +1,6 @@
+from tornado import gen
+
+@gen.coroutine
+def hello():
+    yield gen.sleep(0.001)
+    raise gen.Return("hello")

diff --git a/maint/test/cython/setup.py b/maint/test/cython/setup.py
new file mode 100644 (file)
index 0000000..628bb41
--- /dev/null
+++ b/maint/test/cython/setup.py
@@ -0,0 +1,18 @@
+from setuptools import setup
+
+try:
+    import Cython.Build
+except:
+    Cython = None
+
+if Cython is None:
+    ext_modules = None
+else:
+    ext_modules=Cython.Build.cythonize('cythonapp.pyx')
+
+setup(
+    name='cythonapp',
+    py_modules=['cythonapp_test', 'pythonmodule'],
+    ext_modules=ext_modules,
+    setup_requires='Cython>=0.23.1',
+)

diff --git a/maint/test/cython/tox.ini b/maint/test/cython/tox.ini
new file mode 100644 (file)
index 0000000..38949d5
--- /dev/null
+++ b/maint/test/cython/tox.ini
@@ -0,0 +1,19 @@
+[tox]
+# This currently segfaults on pypy.
+envlist = py27,py32,py33,py34,py35
+
+[testenv]
+deps =
+     ../../..
+     Cython>= 0.23.1
+     backports_abc
+     singledispatch
+commands = python -m unittest cythonapp_test
+# Most of these are defaults, but if you specify any you can't fall back
+# defaults for the others.
+basepython =
+           py27: python2.7
+           py32: python3.2
+           py33: python3.3
+           py34: python3.4
+           py35: python3.5

diff --git a/tornado/concurrent.py b/tornado/concurrent.py
index 7a83b507964b9881140cb1181a13943b03911b7f..d3d878fec257ab4c14aa74e226f581919ce26e5f 100644 (file)
--- a/tornado/concurrent.py
+++ b/tornado/concurrent.py
@@ -177,6 +177,16 @@ class Future(object):
         def __await__(self):
             return (yield self)
         """))
+    else:
+        # Py2-compatible version for use with cython.
+        # Late import of gen.Return to avoid cycles.
+        def __await__(self):
+            result = yield self
+            # StopIteration doesn't take args before py33,
+            # but Cython recognizes the args tuple.
+            e = StopIteration()
+            e.args = (result,)
+            raise e
 
     def cancel(self):
         """Cancel the operation, if possible.

diff --git a/tornado/gen.py b/tornado/gen.py
index 5c7dc019c52c1081e374cdafbc3ece8a2ba97922..34c3c2e0703bc3dd1ef886db249f3e52706f864e 100644 (file)
--- a/tornado/gen.py
+++ b/tornado/gen.py
@@ -101,7 +101,10 @@ except ImportError as e:
 try:
     from collections.abc import Generator as GeneratorType  # py35+
 except ImportError:
-    from types import GeneratorType
+    try:
+        from collections import Generator as GeneratorType  # py2 with backports_abc
+    except ImportError:
+        from types import GeneratorType
 
 try:
     from inspect import isawaitable  # py35+
@@ -139,6 +142,21 @@ class TimeoutError(Exception):
     """Exception raised by ``with_timeout``."""
 
 
+def _value_from_stopiteration(e):
+    try:
+        # StopIteration has a value attribute beginning in py33.
+        # So does our Return class.
+        return e.value
+    except AttributeError:
+        pass
+    try:
+        # Cython backports coroutine functionality by putting the value in
+        # e.args[0].
+        return e.args[0]
+    except (AttributeError, IndexError):
+        return None
+
+
 def engine(func):
     """Callback-oriented decorator for asynchronous generators.
 
@@ -236,7 +254,7 @@ def _make_coroutine_wrapper(func, replace_callback):
         try:
             result = func(*args, **kwargs)
         except (Return, StopIteration) as e:
-            result = getattr(e, 'value', None)
+            result = _value_from_stopiteration(e)
         except Exception:
             future.set_exc_info(sys.exc_info())
             return future
@@ -257,7 +275,7 @@ def _make_coroutine_wrapper(func, replace_callback):
                                 'stack_context inconsistency (probably caused '
                                 'by yield within a "with StackContext" block)'))
                 except (StopIteration, Return) as e:
-                    future.set_result(getattr(e, 'value', None))
+                    future.set_result(_value_from_stopiteration(e))
                 except Exception:
                     future.set_exc_info(sys.exc_info())
                 else:
@@ -302,6 +320,8 @@ class Return(Exception):
     def __init__(self, value=None):
         super(Return, self).__init__()
         self.value = value
+        # Cython recognizes subclasses of StopIteration with a .args tuple.
+        self.args = (value,)
 
 
 class WaitIterator(object):
@@ -946,7 +966,7 @@ class Runner(object):
                         raise LeakedCallbackError(
                             "finished without waiting for callbacks %r" %
                             self.pending_callbacks)
-                    self.result_future.set_result(getattr(e, 'value', None))
+                    self.result_future.set_result(_value_from_stopiteration(e))
                     self.result_future = None
                     self._deactivate_stack_context()
                     return
@@ -1057,11 +1077,57 @@ if sys.version_info >= (3, 3):
     exec(textwrap.dedent("""
     @coroutine
     def _wrap_awaitable(x):
+        if hasattr(x, '__await__'):
+            x = x.__await__()
         return (yield from x)
     """))
 else:
+    # Py2-compatible version for use with Cython.
+    # Copied from PEP 380.
+    @coroutine
     def _wrap_awaitable(x):
-        raise NotImplementedError()
+        if hasattr(x, '__await__'):
+            _i = x.__await__()
+        else:
+            _i = iter(x)
+        try:
+            _y = next(_i)
+        except StopIteration as _e:
+            _r = _value_from_stopiteration(_e)
+        else:
+            while 1:
+                try:
+                    _s = yield _y
+                except GeneratorExit as _e:
+                    try:
+                        _m = _i.close
+                    except AttributeError:
+                        pass
+                    else:
+                        _m()
+                    raise _e
+                except BaseException as _e:
+                    _x = sys.exc_info()
+                    try:
+                        _m = _i.throw
+                    except AttributeError:
+                        raise _e
+                    else:
+                        try:
+                            _y = _m(*_x)
+                        except StopIteration as _e:
+                            _r = _value_from_stopiteration(_e)
+                            break
+                else:
+                    try:
+                        if _s is None:
+                            _y = next(_i)
+                        else:
+                            _y = _i.send(_s)
+                    except StopIteration as _e:
+                        _r = _value_from_stopiteration(_e)
+                        break
+        raise Return(_r)
 
 
 def convert_yielded(yielded):