From: Ben Darnell Date: Sat, 7 Jul 2018 21:29:52 +0000 (-0400) Subject: stack_context: Delete the whole module X-Git-Tag: v6.0.0b1~48^2~5 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=45d9f6d0a23f3ca90d23e00e5c1ed40b955b335d;p=thirdparty%2Ftornado.git stack_context: Delete the whole module --- diff --git a/maint/benchmark/stack_context_benchmark.py b/maint/benchmark/stack_context_benchmark.py deleted file mode 100755 index 2b4a388fe..000000000 --- a/maint/benchmark/stack_context_benchmark.py +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env python -"""Benchmark for stack_context functionality.""" -import collections -import contextlib -import functools -import subprocess -import sys - -from tornado import stack_context - - -class Benchmark(object): - def enter_exit(self, count): - """Measures the overhead of the nested "with" statements - when using many contexts. - """ - if count < 0: - return - with self.make_context(): - self.enter_exit(count - 1) - - def call_wrapped(self, count): - """Wraps and calls a function at each level of stack depth - to measure the overhead of the wrapped function. - """ - # This queue is analogous to IOLoop.add_callback, but lets us - # benchmark the stack_context in isolation without system call - # overhead. - queue = collections.deque() - self.call_wrapped_inner(queue, count) - while queue: - queue.popleft()() - - def call_wrapped_inner(self, queue, count): - if count < 0: - return - with self.make_context(): - queue.append(stack_context.wrap( - functools.partial(self.call_wrapped_inner, queue, count - 1))) - - -class StackBenchmark(Benchmark): - def make_context(self): - return stack_context.StackContext(self.__context) - - @contextlib.contextmanager - def __context(self): - yield - - -class ExceptionBenchmark(Benchmark): - def make_context(self): - return stack_context.ExceptionStackContext(self.__handle_exception) - - def __handle_exception(self, typ, value, tb): - pass - - -def main(): - base_cmd = [ - sys.executable, '-m', 'timeit', '-s', - 'from stack_context_benchmark import StackBenchmark, ExceptionBenchmark'] - cmds = [ - 'StackBenchmark().enter_exit(50)', - 'StackBenchmark().call_wrapped(50)', - 'StackBenchmark().enter_exit(500)', - 'StackBenchmark().call_wrapped(500)', - - 'ExceptionBenchmark().enter_exit(50)', - 'ExceptionBenchmark().call_wrapped(50)', - 'ExceptionBenchmark().enter_exit(500)', - 'ExceptionBenchmark().call_wrapped(500)', - ] - for cmd in cmds: - print(cmd) - subprocess.check_call(base_cmd + [cmd]) - - -if __name__ == '__main__': - main() diff --git a/tornado/curl_httpclient.py b/tornado/curl_httpclient.py index 7f5cb1051..b412897f6 100644 --- a/tornado/curl_httpclient.py +++ b/tornado/curl_httpclient.py @@ -27,7 +27,6 @@ from io import BytesIO from tornado import httputil from tornado import ioloop -from tornado import stack_context from tornado.escape import utf8, native_str from tornado.httpclient import HTTPResponse, HTTPError, AsyncHTTPClient, main @@ -141,17 +140,16 @@ class CurlAsyncHTTPClient(AsyncHTTPClient): def _handle_timeout(self): """Called by IOLoop when the requested timeout has passed.""" - with stack_context.NullContext(): - self._timeout = None - while True: - try: - ret, num_handles = self._multi.socket_action( - pycurl.SOCKET_TIMEOUT, 0) - except pycurl.error as e: - ret = e.args[0] - if ret != pycurl.E_CALL_MULTI_PERFORM: - break - self._finish_pending_requests() + self._timeout = None + while True: + try: + ret, num_handles = self._multi.socket_action( + pycurl.SOCKET_TIMEOUT, 0) + except pycurl.error as e: + ret = e.args[0] + if ret != pycurl.E_CALL_MULTI_PERFORM: + break + self._finish_pending_requests() # In theory, we shouldn't have to do this because curl will # call _set_timeout whenever the timeout changes. However, @@ -174,15 +172,14 @@ class CurlAsyncHTTPClient(AsyncHTTPClient): """Called by IOLoop periodically to ask libcurl to process any events it may have forgotten about. """ - with stack_context.NullContext(): - while True: - try: - ret, num_handles = self._multi.socket_all() - except pycurl.error as e: - ret = e.args[0] - if ret != pycurl.E_CALL_MULTI_PERFORM: - break - self._finish_pending_requests() + while True: + try: + ret, num_handles = self._multi.socket_all() + except pycurl.error as e: + ret = e.args[0] + if ret != pycurl.E_CALL_MULTI_PERFORM: + break + self._finish_pending_requests() def _finish_pending_requests(self): """Process any requests that were completed by the last @@ -199,45 +196,44 @@ class CurlAsyncHTTPClient(AsyncHTTPClient): self._process_queue() def _process_queue(self): - with stack_context.NullContext(): - while True: - started = 0 - while self._free_list and self._requests: - started += 1 - curl = self._free_list.pop() - (request, callback, queue_start_time) = self._requests.popleft() - curl.info = { - "headers": httputil.HTTPHeaders(), - "buffer": BytesIO(), - "request": request, - "callback": callback, - "queue_start_time": queue_start_time, - "curl_start_time": time.time(), - "curl_start_ioloop_time": self.io_loop.current().time(), - } - try: - self._curl_setup_request( - curl, request, curl.info["buffer"], - curl.info["headers"]) - except Exception as e: - # If there was an error in setup, pass it on - # to the callback. Note that allowing the - # error to escape here will appear to work - # most of the time since we are still in the - # caller's original stack frame, but when - # _process_queue() is called from - # _finish_pending_requests the exceptions have - # nowhere to go. - self._free_list.append(curl) - callback(HTTPResponse( - request=request, - code=599, - error=e)) - else: - self._multi.add_handle(curl) - - if not started: - break + while True: + started = 0 + while self._free_list and self._requests: + started += 1 + curl = self._free_list.pop() + (request, callback, queue_start_time) = self._requests.popleft() + curl.info = { + "headers": httputil.HTTPHeaders(), + "buffer": BytesIO(), + "request": request, + "callback": callback, + "queue_start_time": queue_start_time, + "curl_start_time": time.time(), + "curl_start_ioloop_time": self.io_loop.current().time(), + } + try: + self._curl_setup_request( + curl, request, curl.info["buffer"], + curl.info["headers"]) + except Exception as e: + # If there was an error in setup, pass it on + # to the callback. Note that allowing the + # error to escape here will appear to work + # most of the time since we are still in the + # caller's original stack frame, but when + # _process_queue() is called from + # _finish_pending_requests the exceptions have + # nowhere to go. + self._free_list.append(curl) + callback(HTTPResponse( + request=request, + code=599, + error=e)) + else: + self._multi.add_handle(curl) + + if not started: + break def _finish(self, curl, curl_error=None, curl_message=None): info = curl.info diff --git a/tornado/gen.py b/tornado/gen.py index 5bdbca244..8b17568b1 100644 --- a/tornado/gen.py +++ b/tornado/gen.py @@ -103,7 +103,6 @@ from tornado.concurrent import (Future, is_future, chain_future, future_set_exc_ future_add_done_callback, future_set_result_unless_cancelled) from tornado.ioloop import IOLoop from tornado.log import app_log -from tornado import stack_context from tornado.util import TimeoutError @@ -215,14 +214,7 @@ def coroutine(func): # use "optional" coroutines in critical path code without # performance penalty for the synchronous case. try: - orig_stack_contexts = stack_context._state.contexts yielded = next(result) - if stack_context._state.contexts is not orig_stack_contexts: - yielded = _create_future() - yielded.set_exception( - stack_context.StackContextInconsistentError( - 'stack_context inconsistency (probably caused ' - 'by yield within a "with StackContext" block)')) except (StopIteration, Return) as e: future_set_result_unless_cancelled(future, _value_from_stopiteration(e)) except Exception: @@ -577,9 +569,6 @@ def with_timeout(timeout, future, quiet_exceptions=()): Added support for yieldable objects other than `.Future`. """ - # TODO: allow YieldPoints in addition to other yieldables? - # Tricky to do with stack_context semantics. - # # It's tempting to optimize this by cancelling the input future on timeout # instead of creating a new one, but A) we can't know if we are the only # one waiting on the input future, so cancelling it might disrupt other @@ -694,12 +683,6 @@ class Runner(object): self.finished = False self.had_exception = False self.io_loop = IOLoop.current() - # For efficiency, we do not create a stack context until we - # reach a YieldPoint (stack contexts are required for the historical - # semantics of YieldPoints, but not for Futures). When we have - # done so, this field will be set and must be called at the end - # of the coroutine. - self.stack_context_deactivate = None if self.handle_yield(first_yielded): gen = result_future = first_yielded = None self.run() @@ -751,7 +734,6 @@ class Runner(object): return self.future = None try: - orig_stack_contexts = stack_context._state.contexts exc_info = None try: @@ -771,11 +753,6 @@ class Runner(object): else: yielded = self.gen.send(value) - if stack_context._state.contexts is not orig_stack_contexts: - self.gen.throw( - stack_context.StackContextInconsistentError( - 'stack_context inconsistency (probably caused ' - 'by yield within a "with StackContext" block)')) except (StopIteration, Return) as e: self.finished = True self.future = _null_future @@ -790,14 +767,12 @@ class Runner(object): future_set_result_unless_cancelled(self.result_future, _value_from_stopiteration(e)) self.result_future = None - self._deactivate_stack_context() return except Exception: self.finished = True self.future = _null_future future_set_exc_info(self.result_future, sys.exc_info()) self.result_future = None - self._deactivate_stack_context() return if not self.handle_yield(yielded): return @@ -834,11 +809,6 @@ class Runner(object): else: return False - def _deactivate_stack_context(self): - if self.stack_context_deactivate is not None: - self.stack_context_deactivate() - self.stack_context_deactivate = None - # Convert Awaitables into Futures. try: diff --git a/tornado/http1connection.py b/tornado/http1connection.py index d924d621b..fba9cb9bf 100644 --- a/tornado/http1connection.py +++ b/tornado/http1connection.py @@ -29,7 +29,6 @@ from tornado import gen from tornado import httputil from tornado import iostream from tornado.log import gen_log, app_log -from tornado import stack_context from tornado.util import GzipDecompressor @@ -286,7 +285,7 @@ class HTTP1Connection(httputil.HTTPConnection): after sending its request but before receiving all the response. """ - self._close_callback = stack_context.wrap(callback) + self._close_callback = callback def _on_connection_close(self): # Note that this callback is only registered on the IOStream diff --git a/tornado/httpclient.py b/tornado/httpclient.py index abfd8f24c..e3c933058 100644 --- a/tornado/httpclient.py +++ b/tornado/httpclient.py @@ -47,7 +47,7 @@ import weakref from tornado.concurrent import Future, future_set_result_unless_cancelled from tornado.escape import utf8, native_str -from tornado import gen, httputil, stack_context +from tornado import gen, httputil from tornado.ioloop import IOLoop from tornado.util import Configurable @@ -499,38 +499,6 @@ class HTTPRequest(object): def body(self, value): self._body = utf8(value) - @property - def body_producer(self): - return self._body_producer - - @body_producer.setter - def body_producer(self, value): - self._body_producer = stack_context.wrap(value) - - @property - def streaming_callback(self): - return self._streaming_callback - - @streaming_callback.setter - def streaming_callback(self, value): - self._streaming_callback = stack_context.wrap(value) - - @property - def header_callback(self): - return self._header_callback - - @header_callback.setter - def header_callback(self, value): - self._header_callback = stack_context.wrap(value) - - @property - def prepare_curl_callback(self): - return self._prepare_curl_callback - - @prepare_curl_callback.setter - def prepare_curl_callback(self, value): - self._prepare_curl_callback = stack_context.wrap(value) - class HTTPResponse(object): """HTTP Response object. diff --git a/tornado/ioloop.py b/tornado/ioloop.py index 3ce4b1760..2f9319f7b 100644 --- a/tornado/ioloop.py +++ b/tornado/ioloop.py @@ -45,7 +45,6 @@ import random from tornado.concurrent import Future, is_future, chain_future, future_set_exc_info, future_add_done_callback # noqa: E501 from tornado.log import app_log -from tornado import stack_context from tornado.util import Configurable, TimeoutError, unicode_type, import_object @@ -575,25 +574,17 @@ class IOLoop(Configurable): Safe for use from a Python signal handler; should not be used otherwise. - - Callbacks added with this method will be run without any - `.stack_context`, to avoid picking up the context of the function - that was interrupted by the signal. """ raise NotImplementedError() def spawn_callback(self, callback, *args, **kwargs): """Calls the given callback on the next IOLoop iteration. - Unlike all other callback-related methods on IOLoop, - ``spawn_callback`` does not associate the callback with its caller's - ``stack_context``, so it is suitable for fire-and-forget callbacks - that should not interfere with the caller. + As of Tornado 6.0, this method is equivalent to `add_callback`. .. versionadded:: 4.0 """ - with stack_context.NullContext(): - self.add_callback(callback, *args, **kwargs) + self.add_callback(callback, *args, **kwargs) def add_future(self, future, callback): """Schedules a callback on the ``IOLoop`` when the given @@ -607,7 +598,6 @@ class IOLoop(Configurable): interchangeable). """ assert is_future(future) - callback = stack_context.wrap(callback) future_add_done_callback( future, lambda future: self.add_callback(callback, future)) diff --git a/tornado/iostream.py b/tornado/iostream.py index 7d627483e..0df21bc62 100644 --- a/tornado/iostream.py +++ b/tornado/iostream.py @@ -39,7 +39,6 @@ from tornado.concurrent import Future from tornado import ioloop from tornado.log import gen_log, app_log from tornado.netutil import ssl_wrap_socket, _client_ssl_defaults, _server_ssl_defaults -from tornado import stack_context from tornado.util import errno_from_exception try: @@ -540,7 +539,7 @@ class BaseIOStream(object): Unlike other callback-based interfaces, ``set_close_callback`` was not removed in Tornado 6.0. """ - self._close_callback = stack_context.wrap(callback) + self._close_callback = callback self._maybe_add_error_listener() def close(self, exc_info=False): @@ -981,9 +980,8 @@ class BaseIOStream(object): return if self._state is None: self._state = ioloop.IOLoop.ERROR | state - with stack_context.NullContext(): - self.io_loop.add_handler( - self.fileno(), self._handle_events, self._state) + self.io_loop.add_handler( + self.fileno(), self._handle_events, self._state) elif not self._state & state: self._state = self._state | state self.io_loop.update_handler(self.fileno(), self._state) diff --git a/tornado/options.py b/tornado/options.py index 0a4b965f1..f3b87199b 100644 --- a/tornado/options.py +++ b/tornado/options.py @@ -104,7 +104,6 @@ import textwrap from tornado.escape import _unicode, native_str from tornado.log import define_logging_options -from tornado import stack_context from tornado.util import basestring_type, exec_in @@ -421,7 +420,7 @@ class OptionParser(object): def add_parse_callback(self, callback): """Adds a parse callback, to be invoked when option parsing is done.""" - self._parse_callbacks.append(stack_context.wrap(callback)) + self._parse_callbacks.append(callback) def run_parse_callbacks(self): for callback in self._parse_callbacks: diff --git a/tornado/platform/asyncio.py b/tornado/platform/asyncio.py index e0042e1d9..04254bed3 100644 --- a/tornado/platform/asyncio.py +++ b/tornado/platform/asyncio.py @@ -24,7 +24,6 @@ import functools from tornado.gen import convert_yielded from tornado.ioloop import IOLoop -from tornado import stack_context import asyncio @@ -74,7 +73,7 @@ class BaseAsyncIOLoop(IOLoop): fd, fileobj = self.split_fd(fd) if fd in self.handlers: raise ValueError("fd %s added twice" % fd) - self.handlers[fd] = (fileobj, stack_context.wrap(handler)) + self.handlers[fd] = (fileobj, handler) if events & IOLoop.READ: self.asyncio_loop.add_reader( fd, self._handle_events, fd, IOLoop.READ) @@ -142,7 +141,7 @@ class BaseAsyncIOLoop(IOLoop): # convert from absolute to relative. return self.asyncio_loop.call_later( max(0, when - self.time()), self._run_callback, - functools.partial(stack_context.wrap(callback), *args, **kwargs)) + functools.partial(callback, *args, **kwargs)) def remove_timeout(self, timeout): timeout.cancel() @@ -151,7 +150,7 @@ class BaseAsyncIOLoop(IOLoop): try: self.asyncio_loop.call_soon_threadsafe( self._run_callback, - functools.partial(stack_context.wrap(callback), *args, **kwargs)) + functools.partial(callback, *args, **kwargs)) except RuntimeError: # "Event loop is closed". Swallow the exception for # consistency with PollIOLoop (and logical consistency diff --git a/tornado/process.py b/tornado/process.py index 34dd2e084..96f1f4539 100644 --- a/tornado/process.py +++ b/tornado/process.py @@ -33,7 +33,6 @@ from tornado import ioloop from tornado.iostream import PipeIOStream from tornado.log import gen_log from tornado.platform.auto import set_close_exec -from tornado import stack_context from tornado.util import errno_from_exception try: @@ -255,7 +254,7 @@ class Subprocess(object): can be used as an alternative to an exit callback if the signal handler is causing a problem. """ - self._exit_callback = stack_context.wrap(callback) + self._exit_callback = callback Subprocess.initialize() Subprocess._waiting[self.pid] = self Subprocess._try_cleanup_process(self.pid) diff --git a/tornado/simple_httpclient.py b/tornado/simple_httpclient.py index 60b7956fe..851ebb61f 100644 --- a/tornado/simple_httpclient.py +++ b/tornado/simple_httpclient.py @@ -9,7 +9,6 @@ from tornado.ioloop import IOLoop from tornado.iostream import StreamClosedError from tornado.netutil import Resolver, OverrideResolver, _client_ssl_defaults from tornado.log import gen_log -from tornado import stack_context from tornado.tcpclient import TCPClient from tornado.util import PY3 @@ -159,15 +158,14 @@ class SimpleAsyncHTTPClient(AsyncHTTPClient): len(self.active), len(self.queue))) def _process_queue(self): - with stack_context.NullContext(): - while self.queue and len(self.active) < self.max_clients: - key, request, callback = self.queue.popleft() - if key not in self.waiting: - continue - self._remove_timeout(key) - self.active[key] = (request, callback) - release_callback = functools.partial(self._release_fetch, key) - self._handle_request(request, release_callback, callback) + while self.queue and len(self.active) < self.max_clients: + key, request, callback = self.queue.popleft() + if key not in self.waiting: + continue + self._remove_timeout(key) + self.active[key] = (request, callback) + release_callback = functools.partial(self._release_fetch, key) + self._handle_request(request, release_callback, callback) def _connection_class(self): return _HTTPConnection @@ -265,7 +263,7 @@ class _HTTPConnection(httputil.HTTPMessageDelegate): if timeout: self._timeout = self.io_loop.add_timeout( self.start_time + timeout, - stack_context.wrap(functools.partial(self._on_timeout, "while connecting"))) + functools.partial(self._on_timeout, "while connecting")) stream = yield self.tcp_client.connect( host, port, af=af, ssl_options=ssl_options, @@ -283,7 +281,7 @@ class _HTTPConnection(httputil.HTTPMessageDelegate): if self.request.request_timeout: self._timeout = self.io_loop.add_timeout( self.start_time + self.request.request_timeout, - stack_context.wrap(functools.partial(self._on_timeout, "during request"))) + functools.partial(self._on_timeout, "during request")) if (self.request.method not in self._SUPPORTED_METHODS and not self.request.allow_nonstandard_methods): raise KeyError("unknown method %s" % self.request.method) diff --git a/tornado/stack_context.py b/tornado/stack_context.py deleted file mode 100644 index a1eca4c7e..000000000 --- a/tornado/stack_context.py +++ /dev/null @@ -1,413 +0,0 @@ -# -# Copyright 2010 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""`StackContext` allows applications to maintain threadlocal-like state -that follows execution as it moves to other execution contexts. - -The motivating examples are to eliminate the need for explicit -``async_callback`` wrappers (as in `tornado.web.RequestHandler`), and to -allow some additional context to be kept for logging. - -This is slightly magic, but it's an extension of the idea that an -exception handler is a kind of stack-local state and when that stack -is suspended and resumed in a new context that state needs to be -preserved. `StackContext` shifts the burden of restoring that state -from each call site (e.g. wrapping each `.AsyncHTTPClient` callback -in ``async_callback``) to the mechanisms that transfer control from -one context to another (e.g. `.AsyncHTTPClient` itself, `.IOLoop`, -thread pools, etc). - -Example usage:: - - @contextlib.contextmanager - def die_on_error(): - try: - yield - except Exception: - logging.error("exception in asynchronous operation",exc_info=True) - sys.exit(1) - - with StackContext(die_on_error): - # Any exception thrown here *or in callback and its descendants* - # will cause the process to exit instead of spinning endlessly - # in the ioloop. - http_client.fetch(url, callback) - ioloop.start() - -Most applications shouldn't have to work with `StackContext` directly. -Here are a few rules of thumb for when it's necessary: - -* If you're writing an asynchronous library that doesn't rely on a - stack_context-aware library like `tornado.ioloop` or `tornado.iostream` - (for example, if you're writing a thread pool), use - `.stack_context.wrap()` before any asynchronous operations to capture the - stack context from where the operation was started. - -* If you're writing an asynchronous library that has some shared - resources (such as a connection pool), create those shared resources - within a ``with stack_context.NullContext():`` block. This will prevent - ``StackContexts`` from leaking from one request to another. - -* If you want to write something like an exception handler that will - persist across asynchronous calls, create a new `StackContext` (or - `ExceptionStackContext`), and make your asynchronous calls in a ``with`` - block that references your `StackContext`. - -.. deprecated:: 5.1 - - The ``stack_context`` package is deprecated and will be removed in - Tornado 6.0. -""" - -from __future__ import absolute_import, division, print_function - -import sys -import threading -import warnings - -from tornado.util import raise_exc_info - - -class StackContextInconsistentError(Exception): - pass - - -class _State(threading.local): - def __init__(self): - self.contexts = (tuple(), None) - - -_state = _State() - - -class StackContext(object): - """Establishes the given context as a StackContext that will be transferred. - - Note that the parameter is a callable that returns a context - manager, not the context itself. That is, where for a - non-transferable context manager you would say:: - - with my_context(): - - StackContext takes the function itself rather than its result:: - - with StackContext(my_context): - - The result of ``with StackContext() as cb:`` is a deactivation - callback. Run this callback when the StackContext is no longer - needed to ensure that it is not propagated any further (note that - deactivating a context does not affect any instances of that - context that are currently pending). This is an advanced feature - and not necessary in most applications. - """ - def __init__(self, context_factory): - warnings.warn("StackContext is deprecated and will be removed in Tornado 6.0", - DeprecationWarning) - self.context_factory = context_factory - self.contexts = [] - self.active = True - - def _deactivate(self): - self.active = False - - # StackContext protocol - def enter(self): - context = self.context_factory() - self.contexts.append(context) - context.__enter__() - - def exit(self, type, value, traceback): - context = self.contexts.pop() - context.__exit__(type, value, traceback) - - # Note that some of this code is duplicated in ExceptionStackContext - # below. ExceptionStackContext is more common and doesn't need - # the full generality of this class. - def __enter__(self): - self.old_contexts = _state.contexts - self.new_contexts = (self.old_contexts[0] + (self,), self) - _state.contexts = self.new_contexts - - try: - self.enter() - except: - _state.contexts = self.old_contexts - raise - - return self._deactivate - - def __exit__(self, type, value, traceback): - try: - self.exit(type, value, traceback) - finally: - final_contexts = _state.contexts - _state.contexts = self.old_contexts - - # Generator coroutines and with-statements with non-local - # effects interact badly. Check here for signs of - # the stack getting out of sync. - # Note that this check comes after restoring _state.context - # so that if it fails things are left in a (relatively) - # consistent state. - if final_contexts is not self.new_contexts: - raise StackContextInconsistentError( - 'stack_context inconsistency (may be caused by yield ' - 'within a "with StackContext" block)') - - # Break up a reference to itself to allow for faster GC on CPython. - self.new_contexts = None - - -class ExceptionStackContext(object): - """Specialization of StackContext for exception handling. - - The supplied ``exception_handler`` function will be called in the - event of an uncaught exception in this context. The semantics are - similar to a try/finally clause, and intended use cases are to log - an error, close a socket, or similar cleanup actions. The - ``exc_info`` triple ``(type, value, traceback)`` will be passed to the - exception_handler function. - - If the exception handler returns true, the exception will be - consumed and will not be propagated to other exception handlers. - - .. versionadded:: 5.1 - - The ``delay_warning`` argument can be used to delay the emission - of DeprecationWarnings until an exception is caught by the - ``ExceptionStackContext``, which facilitates certain transitional - use cases. - """ - def __init__(self, exception_handler, delay_warning=False): - self.delay_warning = delay_warning - if not self.delay_warning: - warnings.warn( - "StackContext is deprecated and will be removed in Tornado 6.0", - DeprecationWarning) - self.exception_handler = exception_handler - self.active = True - - def _deactivate(self): - self.active = False - - def exit(self, type, value, traceback): - if type is not None: - if self.delay_warning: - warnings.warn( - "StackContext is deprecated and will be removed in Tornado 6.0", - DeprecationWarning) - return self.exception_handler(type, value, traceback) - - def __enter__(self): - self.old_contexts = _state.contexts - self.new_contexts = (self.old_contexts[0], self) - _state.contexts = self.new_contexts - - return self._deactivate - - def __exit__(self, type, value, traceback): - try: - if type is not None: - return self.exception_handler(type, value, traceback) - finally: - final_contexts = _state.contexts - _state.contexts = self.old_contexts - - if final_contexts is not self.new_contexts: - raise StackContextInconsistentError( - 'stack_context inconsistency (may be caused by yield ' - 'within a "with StackContext" block)') - - # Break up a reference to itself to allow for faster GC on CPython. - self.new_contexts = None - - -class NullContext(object): - """Resets the `StackContext`. - - Useful when creating a shared resource on demand (e.g. an - `.AsyncHTTPClient`) where the stack that caused the creating is - not relevant to future operations. - """ - def __enter__(self): - self.old_contexts = _state.contexts - _state.contexts = (tuple(), None) - - def __exit__(self, type, value, traceback): - _state.contexts = self.old_contexts - - -def _remove_deactivated(contexts): - """Remove deactivated handlers from the chain""" - # Clean ctx handlers - stack_contexts = tuple([h for h in contexts[0] if h.active]) - - # Find new head - head = contexts[1] - while head is not None and not head.active: - head = head.old_contexts[1] - - # Process chain - ctx = head - while ctx is not None: - parent = ctx.old_contexts[1] - - while parent is not None: - if parent.active: - break - ctx.old_contexts = parent.old_contexts - parent = parent.old_contexts[1] - - ctx = parent - - return (stack_contexts, head) - - -def wrap(fn): - """Returns a callable object that will restore the current `StackContext` - when executed. - - Use this whenever saving a callback to be executed later in a - different execution context (either in a different thread or - asynchronously in the same thread). - """ - # Check if function is already wrapped - if fn is None or hasattr(fn, '_wrapped'): - return fn - - # Capture current stack head - # TODO: Any other better way to store contexts and update them in wrapped function? - cap_contexts = [_state.contexts] - - if not cap_contexts[0][0] and not cap_contexts[0][1]: - # Fast path when there are no active contexts. - def null_wrapper(*args, **kwargs): - try: - current_state = _state.contexts - _state.contexts = cap_contexts[0] - return fn(*args, **kwargs) - finally: - _state.contexts = current_state - null_wrapper._wrapped = True - return null_wrapper - - def wrapped(*args, **kwargs): - ret = None - try: - # Capture old state - current_state = _state.contexts - - # Remove deactivated items - cap_contexts[0] = contexts = _remove_deactivated(cap_contexts[0]) - - # Force new state - _state.contexts = contexts - - # Current exception - exc = (None, None, None) - top = None - - # Apply stack contexts - last_ctx = 0 - stack = contexts[0] - - # Apply state - for n in stack: - try: - n.enter() - last_ctx += 1 - except: - # Exception happened. Record exception info and store top-most handler - exc = sys.exc_info() - top = n.old_contexts[1] - - # Execute callback if no exception happened while restoring state - if top is None: - try: - ret = fn(*args, **kwargs) - except: - exc = sys.exc_info() - top = contexts[1] - - # If there was exception, try to handle it by going through the exception chain - if top is not None: - exc = _handle_exception(top, exc) - else: - # Otherwise take shorter path and run stack contexts in reverse order - while last_ctx > 0: - last_ctx -= 1 - c = stack[last_ctx] - - try: - c.exit(*exc) - except: - exc = sys.exc_info() - top = c.old_contexts[1] - break - else: - top = None - - # If if exception happened while unrolling, take longer exception handler path - if top is not None: - exc = _handle_exception(top, exc) - - # If exception was not handled, raise it - if exc != (None, None, None): - raise_exc_info(exc) - finally: - _state.contexts = current_state - return ret - - wrapped._wrapped = True - return wrapped - - -def _handle_exception(tail, exc): - while tail is not None: - try: - if tail.exit(*exc): - exc = (None, None, None) - except: - exc = sys.exc_info() - - tail = tail.old_contexts[1] - - return exc - - -def run_with_stack_context(context, func): - """Run a coroutine ``func`` in the given `StackContext`. - - It is not safe to have a ``yield`` statement within a ``with StackContext`` - block, so it is difficult to use stack context with `.gen.coroutine`. - This helper function runs the function in the correct context while - keeping the ``yield`` and ``with`` statements syntactically separate. - - Example:: - - @gen.coroutine - def incorrect(): - with StackContext(ctx): - # ERROR: this will raise StackContextInconsistentError - yield other_coroutine() - - @gen.coroutine - def correct(): - yield run_with_stack_context(StackContext(ctx), other_coroutine) - - .. versionadded:: 3.1 - """ - with context: - return func() diff --git a/tornado/test/concurrent_test.py b/tornado/test/concurrent_test.py index 99402d167..05a419c4d 100644 --- a/tornado/test/concurrent_test.py +++ b/tornado/test/concurrent_test.py @@ -23,7 +23,6 @@ from tornado.concurrent import Future, run_on_executor, future_set_result_unless from tornado.escape import utf8, to_unicode from tornado import gen from tornado.iostream import IOStream -from tornado import stack_context from tornado.tcpserver import TCPServer from tornado.testing import AsyncTestCase, bind_unused_port, gen_test from tornado.test.util import unittest, skipBefore35, exec_test diff --git a/tornado/test/curl_httpclient_test.py b/tornado/test/curl_httpclient_test.py index 347bde641..52c42c8b1 100644 --- a/tornado/test/curl_httpclient_test.py +++ b/tornado/test/curl_httpclient_test.py @@ -4,12 +4,9 @@ from __future__ import absolute_import, division, print_function from hashlib import md5 from tornado.escape import utf8 -from tornado.httpclient import HTTPRequest, HTTPClientError -from tornado.locks import Event -from tornado.stack_context import ExceptionStackContext -from tornado.testing import AsyncHTTPTestCase, gen_test +from tornado.testing import AsyncHTTPTestCase from tornado.test import httpclient_test -from tornado.test.util import unittest, ignore_deprecation +from tornado.test.util import unittest from tornado.web import Application, RequestHandler @@ -101,24 +98,6 @@ class CurlHTTPClientTestCase(AsyncHTTPTestCase): defaults=dict(allow_ipv6=False), **kwargs) - @gen_test - def test_prepare_curl_callback_stack_context(self): - exc_info = [] - error_event = Event() - - def error_handler(typ, value, tb): - exc_info.append((typ, value, tb)) - error_event.set() - return True - - with ignore_deprecation(): - with ExceptionStackContext(error_handler): - request = HTTPRequest(self.get_url('/custom_reason'), - prepare_curl_callback=lambda curl: 1 / 0) - yield [error_event.wait(), self.http_client.fetch(request)] - self.assertEqual(1, len(exc_info)) - self.assertIs(exc_info[0][0], ZeroDivisionError) - def test_digest_auth(self): response = self.fetch('/digest', auth_mode='digest', auth_username='foo', auth_password='bar') diff --git a/tornado/test/httpclient_test.py b/tornado/test/httpclient_test.py index d994ab8d0..882e0faff 100644 --- a/tornado/test/httpclient_test.py +++ b/tornado/test/httpclient_test.py @@ -5,7 +5,6 @@ import base64 import binascii from contextlib import closing import copy -import sys import threading import datetime from io import BytesIO @@ -20,9 +19,8 @@ from tornado.ioloop import IOLoop from tornado.iostream import IOStream from tornado.log import gen_log from tornado import netutil -from tornado.stack_context import ExceptionStackContext, NullContext from tornado.testing import AsyncHTTPTestCase, bind_unused_port, gen_test, ExpectLog -from tornado.test.util import unittest, skipOnTravis, ignore_deprecation +from tornado.test.util import unittest, skipOnTravis from tornado.web import Application, RequestHandler, url from tornado.httputil import format_timestamp, HTTPHeaders @@ -218,27 +216,6 @@ Transfer-Encoding: chunked self.assertEqual(resp.body, b"12") self.io_loop.remove_handler(sock.fileno()) - def test_streaming_stack_context(self): - chunks = [] - exc_info = [] - - def error_handler(typ, value, tb): - exc_info.append((typ, value, tb)) - return True - - def streaming_cb(chunk): - chunks.append(chunk) - if chunk == b'qwer': - 1 / 0 - - with ignore_deprecation(): - with ExceptionStackContext(error_handler): - self.fetch('/chunk', streaming_callback=streaming_cb) - - self.assertEqual(chunks, [b'asdf', b'qwer']) - self.assertEqual(1, len(exc_info)) - self.assertIs(exc_info[0][0], ZeroDivisionError) - def test_basic_auth(self): # This test data appears in section 2 of RFC 7617. self.assertEqual(self.fetch("/auth", auth_username="Aladdin", @@ -352,23 +329,6 @@ Transfer-Encoding: chunked self.assertRegexpMatches(first_line[0], 'HTTP/[0-9]\\.[0-9] 200.*\r\n') self.assertEqual(chunks, [b'asdf', b'qwer']) - def test_header_callback_stack_context(self): - exc_info = [] - - def error_handler(typ, value, tb): - exc_info.append((typ, value, tb)) - return True - - def header_callback(header_line): - if header_line.lower().startswith('content-type:'): - 1 / 0 - - with ignore_deprecation(): - with ExceptionStackContext(error_handler): - self.fetch('/chunk', header_callback=header_callback) - self.assertEqual(len(exc_info), 1) - self.assertIs(exc_info[0][0], ZeroDivisionError) - @gen_test def test_configure_defaults(self): defaults = dict(user_agent='TestDefaultUserAgent', allow_ipv6=False) diff --git a/tornado/test/import_test.py b/tornado/test/import_test.py index ecc569eb5..42668d64b 100644 --- a/tornado/test/import_test.py +++ b/tornado/test/import_test.py @@ -35,7 +35,6 @@ import tornado.netutil import tornado.options import tornado.process import tornado.simple_httpclient -import tornado.stack_context import tornado.tcpserver import tornado.tcpclient import tornado.template diff --git a/tornado/test/ioloop_test.py b/tornado/test/ioloop_test.py index bc42bef64..60adaca26 100644 --- a/tornado/test/ioloop_test.py +++ b/tornado/test/ioloop_test.py @@ -22,10 +22,9 @@ from tornado.escape import native_str from tornado import gen from tornado.ioloop import IOLoop, TimeoutError, PeriodicCallback from tornado.log import app_log -from tornado.stack_context import ExceptionStackContext, StackContext, wrap, NullContext from tornado.testing import AsyncTestCase, bind_unused_port, ExpectLog, gen_test from tornado.test.util import (unittest, skipIfNonUnix, skipOnTravis, - skipBefore35, exec_test, ignore_deprecation) + skipBefore35, exec_test) try: from concurrent import futures @@ -332,24 +331,20 @@ class TestIOLoop(AsyncTestCase): def test_exception_logging(self): """Uncaught exceptions get logged by the IOLoop.""" - # Use a NullContext to keep the exception from being caught by - # AsyncTestCase. - with NullContext(): - self.io_loop.add_callback(lambda: 1 / 0) - self.io_loop.add_callback(self.stop) - with ExpectLog(app_log, "Exception in callback"): - self.wait() + self.io_loop.add_callback(lambda: 1 / 0) + self.io_loop.add_callback(self.stop) + with ExpectLog(app_log, "Exception in callback"): + self.wait() def test_exception_logging_future(self): """The IOLoop examines exceptions from Futures and logs them.""" - with NullContext(): - @gen.coroutine - def callback(): - self.io_loop.add_callback(self.stop) - 1 / 0 - self.io_loop.add_callback(callback) - with ExpectLog(app_log, "Exception in callback"): - self.wait() + @gen.coroutine + def callback(): + self.io_loop.add_callback(self.stop) + 1 / 0 + self.io_loop.add_callback(callback) + with ExpectLog(app_log, "Exception in callback"): + self.wait() @skipBefore35 def test_exception_logging_native_coro(self): @@ -361,24 +356,23 @@ class TestIOLoop(AsyncTestCase): self.io_loop.add_callback(self.io_loop.add_callback, self.stop) 1 / 0 """) - with NullContext(): - self.io_loop.add_callback(namespace["callback"]) - with ExpectLog(app_log, "Exception in callback"): - self.wait() + self.io_loop.add_callback(namespace["callback"]) + with ExpectLog(app_log, "Exception in callback"): + self.wait() def test_spawn_callback(self): - with ignore_deprecation(): - # An added callback runs in the test's stack_context, so will be - # re-raised in wait(). - self.io_loop.add_callback(lambda: 1 / 0) - with self.assertRaises(ZeroDivisionError): - self.wait() - # A spawned callback is run directly on the IOLoop, so it will be - # logged without stopping the test. - self.io_loop.spawn_callback(lambda: 1 / 0) - self.io_loop.add_callback(self.stop) - with ExpectLog(app_log, "Exception in callback"): - self.wait() + # Both add_callback and spawn_callback run directly on the IOLoop, + # so their errors are logged without stopping the test. + self.io_loop.add_callback(lambda: 1 / 0) + self.io_loop.add_callback(self.stop) + with ExpectLog(app_log, "Exception in callback"): + self.wait() + # A spawned callback is run directly on the IOLoop, so it will be + # logged without stopping the test. + self.io_loop.spawn_callback(lambda: 1 / 0) + self.io_loop.add_callback(self.stop) + with ExpectLog(app_log, "Exception in callback"): + self.wait() @skipIfNonUnix def test_remove_handler_from_handler(self): @@ -475,64 +469,6 @@ class TestIOLoopCurrentAsync(AsyncTestCase): yield e.submit(IOLoop.clear_current) -class TestIOLoopAddCallback(AsyncTestCase): - def setUp(self): - super(TestIOLoopAddCallback, self).setUp() - self.active_contexts = [] - - def add_callback(self, callback, *args, **kwargs): - self.io_loop.add_callback(callback, *args, **kwargs) - - @contextlib.contextmanager - def context(self, name): - self.active_contexts.append(name) - yield - self.assertEqual(self.active_contexts.pop(), name) - - def test_pre_wrap(self): - # A pre-wrapped callback is run in the context in which it was - # wrapped, not when it was added to the IOLoop. - def f1(): - self.assertIn('c1', self.active_contexts) - self.assertNotIn('c2', self.active_contexts) - self.stop() - - with ignore_deprecation(): - with StackContext(functools.partial(self.context, 'c1')): - wrapped = wrap(f1) - - with StackContext(functools.partial(self.context, 'c2')): - self.add_callback(wrapped) - - self.wait() - - def test_pre_wrap_with_args(self): - # Same as test_pre_wrap, but the function takes arguments. - # Implementation note: The function must not be wrapped in a - # functools.partial until after it has been passed through - # stack_context.wrap - def f1(foo, bar): - self.assertIn('c1', self.active_contexts) - self.assertNotIn('c2', self.active_contexts) - self.stop((foo, bar)) - - with ignore_deprecation(): - with StackContext(functools.partial(self.context, 'c1')): - wrapped = wrap(f1) - - with StackContext(functools.partial(self.context, 'c2')): - self.add_callback(wrapped, 1, bar=2) - - result = self.wait() - self.assertEqual(result, (1, 2)) - - -class TestIOLoopAddCallbackFromSignal(TestIOLoopAddCallback): - # Repeat the add_callback tests using add_callback_from_signal - def add_callback(self, callback, *args, **kwargs): - self.io_loop.add_callback_from_signal(callback, *args, **kwargs) - - @unittest.skipIf(futures is None, "futures module not present") class TestIOLoopFutures(AsyncTestCase): def test_add_future_threads(self): @@ -543,39 +479,6 @@ class TestIOLoopFutures(AsyncTestCase): self.assertTrue(future.done()) self.assertTrue(future.result() is None) - def test_add_future_stack_context(self): - ready = threading.Event() - - def task(): - # we must wait for the ioloop callback to be scheduled before - # the task completes to ensure that add_future adds the callback - # asynchronously (which is the scenario in which capturing - # the stack_context matters) - ready.wait(1) - assert ready.isSet(), "timed out" - raise Exception("worker") - - def callback(future): - self.future = future - raise Exception("callback") - - def handle_exception(typ, value, traceback): - self.exception = value - self.stop() - return True - - # stack_context propagates to the ioloop callback, but the worker - # task just has its exceptions caught and saved in the Future. - with ignore_deprecation(): - with futures.ThreadPoolExecutor(1) as pool: - with ExceptionStackContext(handle_exception): - self.io_loop.add_future(pool.submit(task), callback) - ready.set() - self.wait() - - self.assertEqual(self.exception.args[0], "callback") - self.assertEqual(self.future.exception().args[0], "worker") - @gen_test def test_run_in_executor_gen(self): event1 = threading.Event() diff --git a/tornado/test/tcpserver_test.py b/tornado/test/tcpserver_test.py index 4c238a321..09d8ba678 100644 --- a/tornado/test/tcpserver_test.py +++ b/tornado/test/tcpserver_test.py @@ -9,7 +9,6 @@ from tornado.escape import utf8, to_unicode from tornado import gen from tornado.iostream import IOStream from tornado.log import app_log -from tornado.stack_context import NullContext from tornado.tcpserver import TCPServer from tornado.test.util import skipBefore35, skipIfNonUnix, exec_test, unittest from tornado.testing import AsyncTestCase, ExpectLog, bind_unused_port, gen_test @@ -30,9 +29,8 @@ class TCPServerTest(AsyncTestCase): server = client = None try: sock, port = bind_unused_port() - with NullContext(): - server = TestServer() - server.add_socket(sock) + server = TestServer() + server.add_socket(sock) client = IOStream(socket.socket()) with ExpectLog(app_log, "Exception in callback"): yield client.connect(('localhost', port)) diff --git a/tornado/test/testing_test.py b/tornado/test/testing_test.py index d9695c777..97644f60e 100644 --- a/tornado/test/testing_test.py +++ b/tornado/test/testing_test.py @@ -2,10 +2,8 @@ from __future__ import absolute_import, division, print_function from tornado import gen, ioloop from tornado.httpserver import HTTPServer -from tornado.log import app_log -from tornado.simple_httpclient import SimpleAsyncHTTPClient, HTTPTimeoutError -from tornado.test.util import unittest, skipBefore35, exec_test, ignore_deprecation -from tornado.testing import AsyncHTTPTestCase, AsyncTestCase, bind_unused_port, gen_test, ExpectLog +from tornado.test.util import unittest, skipBefore35, exec_test +from tornado.testing import AsyncHTTPTestCase, AsyncTestCase, bind_unused_port, gen_test from tornado.web import Application import contextlib import os @@ -34,15 +32,6 @@ def set_environ(name, value): class AsyncTestCaseTest(AsyncTestCase): - def test_exception_in_callback(self): - with ignore_deprecation(): - self.io_loop.add_callback(lambda: 1 / 0) - try: - self.wait() - self.fail("did not get expected exception") - except ZeroDivisionError: - pass - def test_wait_timeout(self): time = self.io_loop.time @@ -71,18 +60,6 @@ class AsyncTestCaseTest(AsyncTestCase): self.io_loop.add_timeout(self.io_loop.time() + 0.03, self.stop) self.wait(timeout=0.15) - def test_multiple_errors(self): - with ignore_deprecation(): - def fail(message): - raise Exception(message) - self.io_loop.add_callback(lambda: fail("error one")) - self.io_loop.add_callback(lambda: fail("error two")) - # The first error gets raised; the second gets logged. - with ExpectLog(app_log, "multiple unhandled exceptions"): - with self.assertRaises(Exception) as cm: - self.wait() - self.assertEqual(str(cm.exception), "error one") - class AsyncHTTPTestCaseTest(AsyncHTTPTestCase): def setUp(self): diff --git a/tornado/test/web_test.py b/tornado/test/web_test.py index d3126ab3a..c555fd7af 100644 --- a/tornado/test/web_test.py +++ b/tornado/test/web_test.py @@ -5,7 +5,6 @@ from tornado import gen from tornado.escape import json_decode, utf8, to_unicode, recursive_unicode, native_str, to_basestring # noqa: E501 from tornado.httpclient import HTTPClientError from tornado.httputil import format_timestamp -from tornado.ioloop import IOLoop from tornado.iostream import IOStream from tornado import locale from tornado.locks import Event @@ -13,7 +12,7 @@ from tornado.log import app_log, gen_log from tornado.simple_httpclient import SimpleAsyncHTTPClient from tornado.template import DictLoader from tornado.testing import AsyncHTTPTestCase, AsyncTestCase, ExpectLog, gen_test -from tornado.test.util import unittest, skipBefore35, exec_test, ignore_deprecation +from tornado.test.util import unittest, skipBefore35, exec_test from tornado.util import ObjectDict, unicode_type, PY3 from tornado.web import ( Application, RequestHandler, StaticFileHandler, RedirectHandler as WebRedirectHandler, diff --git a/tornado/testing.py b/tornado/testing.py index 86cbf128b..018374fc6 100644 --- a/tornado/testing.py +++ b/tornado/testing.py @@ -30,7 +30,6 @@ from tornado import netutil from tornado.platform.asyncio import AsyncIOMainLoop from tornado.process import Subprocess from tornado.log import app_log -from tornado.stack_context import ExceptionStackContext from tornado.util import raise_exc_info, basestring_type, PY3 @@ -206,8 +205,7 @@ class AsyncTestCase(unittest.TestCase): raise_exc_info(failure) def run(self, result=None): - with ExceptionStackContext(self._handle_exception, delay_warning=True): - super(AsyncTestCase, self).run(result) + super(AsyncTestCase, self).run(result) # As a last resort, if an exception escaped super.run() and wasn't # re-raised in tearDown, raise it here. This will cause the # unittest run to fail messily, but that's better than silently @@ -601,7 +599,7 @@ def main(**kwargs): The easiest way to run a test is via the command line:: - python -m tornado.testing tornado.test.stack_context_test + python -m tornado.testing tornado.test.web_test See the standard library unittest module for ways in which tests can be specified. @@ -616,7 +614,7 @@ def main(**kwargs): # Runs all tests python -m tornado.test.runtests # Runs one test - python -m tornado.test.runtests tornado.test.stack_context_test + python -m tornado.test.runtests tornado.test.web_test Additional keyword arguments passed through to ``unittest.main()``. For example, use ``tornado.testing.main(verbosity=2)`` diff --git a/tornado/web.py b/tornado/web.py index b483796a3..49639b415 100644 --- a/tornado/web.py +++ b/tornado/web.py @@ -78,7 +78,6 @@ import time import tornado import traceback import types -import warnings from inspect import isclass from io import BytesIO @@ -89,13 +88,12 @@ from tornado import httputil from tornado import iostream from tornado import locale from tornado.log import access_log, app_log, gen_log -from tornado import stack_context from tornado import template from tornado.escape import utf8, _unicode from tornado.routing import (AnyMatches, DefaultHostMatches, HostMatches, ReversibleRouter, Rule, ReversibleRuleRouter, URLSpec) -from tornado.util import (ObjectDict, raise_exc_info, +from tornado.util import (ObjectDict, unicode_type, _websocket_mask, PY3) url = URLSpec @@ -1538,17 +1536,6 @@ class RequestHandler(object): break return match - def _stack_context_handle_exception(self, type, value, traceback): - try: - # For historical reasons _handle_request_exception only takes - # the exception value instead of the full triple, - # so re-raise the exception to ensure that it's in - # sys.exc_info() - raise_exc_info((type, value, traceback)) - except Exception: - self._handle_request_exception(value) - return True - @gen.coroutine def _execute(self, transforms, *args, **kwargs): """Executes this request with the given output transforms."""