From: Ben Darnell Date: Sat, 29 Sep 2012 22:55:47 +0000 (-0700) Subject: Merge branch 'master' into futures X-Git-Tag: v3.0.0~263^2~1 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=aaf8c3687e00550e66179be6c3bb834d2246d1c2;p=thirdparty%2Ftornado.git Merge branch 'master' into futures Conflicts: tornado/autoreload.py tornado/ioloop.py tornado/netutil.py tornado/simple_httpclient.py tornado/test/ioloop_test.py tornado/test/runtests.py tox.ini --- aaf8c3687e00550e66179be6c3bb834d2246d1c2 diff --cc tornado/autoreload.py index 940c2be7a,27ef1d103..cdb2f124f --- a/tornado/autoreload.py +++ b/tornado/autoreload.py @@@ -273,22 -273,16 +274,25 @@@ def main() # module) will see the right things. exec f.read() in globals(), globals() except SystemExit, e: - logging.info("Script exited with status %s", e.code) + logging.basicConfig() + gen_log.info("Script exited with status %s", e.code) except Exception, e: - logging.warning("Script exited with uncaught exception", exc_info=True) + logging.basicConfig() + gen_log.warning("Script exited with uncaught exception", exc_info=True) + # If an exception occurred at import time, the file with the error + # never made it into sys.modules and so we won't know to watch it. + # Just to make sure we've covered everything, walk the stack trace + # from the exception and watch every file. + for (filename, lineno, name, line) in traceback.extract_tb(sys.exc_info()[2]): + watch(filename) if isinstance(e, SyntaxError): + # SyntaxErrors are special: their innermost stack frame is fake + # so extract_tb won't see it and we have to get the filename + # from the exception object. watch(e.filename) else: - logging.info("Script exited normally") + logging.basicConfig() + gen_log.info("Script exited normally") # restore sys.argv so subsequent executions will include autoreload sys.argv = original_argv diff --cc tornado/ioloop.py index 724472092,2d4971854..236feaa41 --- a/tornado/ioloop.py +++ b/tornado/ioloop.py @@@ -30,17 -30,16 +30,18 @@@ from __future__ import absolute_import import datetime import errno +import functools import heapq - import os import logging + import os import select import thread import threading import time import traceback +from tornado.concurrent import DummyFuture + from tornado.log import app_log, gen_log from tornado import stack_context try: @@@ -280,10 -267,40 +290,42 @@@ class IOLoop(object) if self._stopped: self._stopped = False return + old_current = getattr(IOLoop._current, "instance", None) + IOLoop._current.instance = self self._thread_ident = thread.get_ident() self._running = True + + # signal.set_wakeup_fd closes a race condition in event loops: + # a signal may arrive at the beginning of select/poll/etc + # before it goes into its interruptible sleep, so the signal + # will be consumed without waking the select. The solution is + # for the (C, synchronous) signal handler to write to a pipe, + # which will then be seen by select. + # + # In python's signal handling semantics, this only matters on the + # main thread (fortunately, set_wakeup_fd only works on the main + # thread and will raise a ValueError otherwise). + # + # If someone has already set a wakeup fd, we don't want to + # disturb it. This is an issue for twisted, which does its + # SIGCHILD processing in response to its own wakeup fd being + # written to. As long as the wakeup fd is registered on the IOLoop, + # the loop will still wake up and everything should work. + old_wakeup_fd = None + if hasattr(signal, 'set_wakeup_fd') and os.name == 'posix': + # requires python 2.6+, unix. set_wakeup_fd exists but crashes + # the python process on windows. + try: + old_wakeup_fd = signal.set_wakeup_fd(self._waker.write_fileno()) + if old_wakeup_fd != -1: + # Already set, restore previous value. This is a little racy, + # but there's no clean get_wakeup_fd and in real use the + # IOLoop is just started once at the beginning. + signal.set_wakeup_fd(old_wakeup_fd) + old_wakeup_fd = None + except ValueError: # non-main thread + pass + while True: poll_timeout = 3600.0 @@@ -364,7 -381,8 +406,9 @@@ self._stopped = False if self._blocking_signal_threshold is not None: signal.setitimer(signal.ITIMER_REAL, 0, 0) + IOLoop._current.instance = old_current + if old_wakeup_fd is not None: + signal.set_wakeup_fd(old_wakeup_fd) def stop(self): """Stop the loop after the current event loop iteration is complete. @@@ -442,19 -464,32 +490,45 @@@ # avoid it when we can. self._waker.wake() + def add_callback_from_signal(self, callback): + """Calls the given callback on the next I/O loop iteration. + + Safe for use from a Python signal handler; should not be used + otherwise. + + Callbacks added with this method will be run without any + stack_context, to avoid picking up the context of the function + that was interrupted by the signal. + """ + with stack_context.NullContext(): + if thread.get_ident() != self._thread_ident: + # if the signal is handled on another thread, we can add + # it normally (modulo the NullContext) + self.add_callback(callback) + else: + # If we're on the IOLoop's thread, we cannot use + # the regular add_callback because it may deadlock on + # _callback_lock. Blindly insert into self._callbacks. + # This is safe because the GIL makes list.append atomic. + # One subtlety is that if the signal interrupted the + # _callback_lock block in IOLoop.start, we may modify + # either the old or new version of self._callbacks, + # but either way will work. + self._callbacks.append(stack_context.wrap(callback)) + + if futures is not None: + _FUTURE_TYPES = (futures.Future, DummyFuture) + else: + _FUTURE_TYPES = DummyFuture + def add_future(self, future, callback): + """Schedules a callback on the IOLoop when the given future is finished. + """ + assert isinstance(future, IOLoop._FUTURE_TYPES) + callback = stack_context.wrap(callback) + future.add_done_callback( + lambda future: self.add_callback( + functools.partial(callback, future))) + def _run_callback(self, callback): try: callback() diff --cc tornado/netutil.py index ff66d987b,d796a2ae2..291dbecd9 --- a/tornado/netutil.py +++ b/tornado/netutil.py @@@ -26,9 -24,9 +24,10 @@@ import socke import stat from tornado import process - from tornado.concurrent import DummyFuture, dummy_executor, run_on_executor ++from tornado.concurrent import dummy_executor, run_on_executor from tornado.ioloop import IOLoop from tornado.iostream import IOStream, SSLIOStream + from tornado.log import app_log from tornado.platform.auto import set_close_exec try: diff --cc tornado/simple_httpclient.py index 67107de4b,ae2988424..cb894468f --- a/tornado/simple_httpclient.py +++ b/tornado/simple_httpclient.py @@@ -5,7 -5,7 +5,8 @@@ from tornado.escape import utf8, _unico from tornado.httpclient import HTTPRequest, HTTPResponse, HTTPError, AsyncHTTPClient, main from tornado.httputil import HTTPHeaders from tornado.iostream import IOStream, SSLIOStream +from tornado.netutil import Resolver + from tornado.log import gen_log from tornado import stack_context from tornado.util import b, GzipDecompressor diff --cc tornado/test/ioloop_test.py index bfdc6ca58,8bfffe0ac..eec24c48f --- a/tornado/test/ioloop_test.py +++ b/tornado/test/ioloop_test.py @@@ -8,18 -7,11 +7,17 @@@ import threadin import time from tornado.ioloop import IOLoop - from tornado.netutil import bind_sockets +from tornado.stack_context import ExceptionStackContext - from tornado.testing import AsyncTestCase, LogTrapTestCase, get_unused_port + from tornado.testing import AsyncTestCase, bind_unused_port from tornado.test.util import unittest +try: + from concurrent import futures +except ImportError: + futures = None + - class TestIOLoop(AsyncTestCase, LogTrapTestCase): + class TestIOLoop(AsyncTestCase): def test_add_callback_wakeup(self): # Make sure that add_callback from inside a running IOLoop # wakes up the IOLoop immediately instead of waiting for a timeout. @@@ -55,47 -58,23 +64,64 @@@ finally: sock.close() + def test_add_callback_from_signal(self): + # cheat a little bit and just run this normally, since we can't + # easily simulate the races that happen with real signal handlers + self.io_loop.add_callback_from_signal(self.stop) + self.wait() + + def test_add_callback_from_signal_other_thread(self): + # Very crude test, just to make sure that we cover this case. + # This also happens to be the first test where we run an IOLoop in + # a non-main thread. + other_ioloop = IOLoop() + thread = threading.Thread(target=other_ioloop.start) + thread.start() + other_ioloop.add_callback_from_signal(other_ioloop.stop) + thread.join() + other_ioloop.close() + - class TestIOLoopFutures(AsyncTestCase, LogTrapTestCase): ++class TestIOLoopFutures(AsyncTestCase): + def test_add_future_threads(self): + with futures.ThreadPoolExecutor(1) as pool: + self.io_loop.add_future(pool.submit(lambda: None), + lambda future: self.stop(future)) + future = self.wait() + self.assertTrue(future.done()) + self.assertTrue(future.result() is None) + + def test_add_future_stack_context(self): + ready = threading.Event() + def task(): + # we must wait for the ioloop callback to be scheduled before + # the task completes to ensure that add_future adds the callback + # asynchronously (which is the scenario in which capturing + # the stack_context matters) + ready.wait(1) + assert ready.isSet(), "timed out" + raise Exception("worker") + def callback(future): + self.future = future + raise Exception("callback") + def handle_exception(typ, value, traceback): + self.exception = value + self.stop() + return True + + # stack_context propagates to the ioloop callback, but the worker + # task just has its exceptions caught and saved in the Future. + with futures.ThreadPoolExecutor(1) as pool: + with ExceptionStackContext(handle_exception): + self.io_loop.add_future(pool.submit(task), callback) + ready.set() + self.wait() + + self.assertEqual(self.exception.args[0], "callback") + self.assertEqual(self.future.exception().args[0], "worker") +TestIOLoopFutures = unittest.skipIf( + futures is None, "futures module not present")(TestIOLoopFutures) + + if __name__ == "__main__": unittest.main() diff --cc tornado/test/runtests.py index 174554989,adf761db4..1f8e43ea7 --- a/tornado/test/runtests.py +++ b/tornado/test/runtests.py @@@ -21,7 -21,7 +22,8 @@@ TEST_MODULES = 'tornado.test.ioloop_test', 'tornado.test.iostream_test', 'tornado.test.locale_test', + 'tornado.test.netutil_test', + 'tornado.test.log_test', 'tornado.test.options_test', 'tornado.test.process_test', 'tornado.test.simple_httpclient_test', @@@ -66,12 -66,9 +68,14 @@@ if __name__ == '__main__' warnings.filterwarnings("ignore", category=DeprecationWarning) warnings.filterwarnings("error", category=DeprecationWarning, module=r"tornado\..*") + # The unittest module is aggressive about deprecating redundant methods, + # leaving some without non-deprecated spellings that work on both + # 2.7 and 3.2 + warnings.filterwarnings("ignore", category=DeprecationWarning, + message="Please use assert.* instead") + logging.getLogger("tornado.access").setLevel(logging.CRITICAL) + import tornado.testing kwargs = {} if sys.version_info >= (3, 2): diff --cc tox.ini index 4b73a1aa5,a735049da..677a7411c --- a/tox.ini +++ b/tox.ini @@@ -35,8 -33,6 +33,7 @@@ deps [testenv:py25-full] basepython = python2.5 deps = + futures - MySQL-python pycurl simplejson # twisted is dropping python 2.5 support in 12.2.0 @@@ -54,8 -50,6 +51,7 @@@ deps = unittest [testenv:py26-full] basepython = python2.6 deps = + futures - MySQL-python pycurl twisted==11.0.0 unittest2 @@@ -63,8 -57,6 +59,7 @@@ [testenv:py27-full] basepython = python2.7 deps = + futures - MySQL-python pycurl twisted>=12.0.0 @@@ -74,8 -66,6 +69,7 @@@ # this flag controls which client all the other tests use. basepython = python2.7 deps = + futures - MySQL-python pycurl twisted>=11.1.0 commands = python -m tornado.test.runtests --httpclient=tornado.curl_httpclient.CurlAsyncHTTPClient {posargs:} @@@ -87,8 -77,6 +81,7 @@@ # run this configuration there. basepython = pypy deps = + futures - MySQL-python twisted>=12.1.0 # In python 3, opening files in text mode uses a system-dependent encoding by @@@ -118,8 -106,6 +111,7 @@@ basepython = python3. [testenv:py27-opt] basepython = python2.7 deps = + futures - MySQL-python pycurl twisted>=12.0.0 commands = python -O -m tornado.test.runtests {posargs:}