From: Ben Darnell Date: Mon, 21 Jan 2013 00:58:35 +0000 (-0500) Subject: Move TCPServer to its own module. X-Git-Tag: v3.0.0~160 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=b3d8cd307491323e5cbb6d0ca182744ac12c55c6;p=thirdparty%2Ftornado.git Move TCPServer to its own module. It's backwards-incompatible for people who used it directly, but that's what 3.0 is for. ;) TCPServer made it impossible to import netutil from iostream due to cyclic imports, which seems backwards. --- diff --git a/tornado/httpserver.py b/tornado/httpserver.py index 11ac8a9c2..83babaac8 100644 --- a/tornado/httpserver.py +++ b/tornado/httpserver.py @@ -34,7 +34,7 @@ from tornado.escape import native_str, parse_qs_bytes from tornado import httputil from tornado import iostream from tornado.log import gen_log -from tornado.netutil import TCPServer +from tornado.tcpserver import TCPServer from tornado import stack_context from tornado.util import bytes_type @@ -103,9 +103,9 @@ class HTTPServer(TCPServer): }) `HTTPServer` initialization follows one of three patterns (the - initialization methods are defined on `tornado.netutil.TCPServer`): + initialization methods are defined on `tornado.tcpserver.TCPServer`): - 1. `~tornado.netutil.TCPServer.listen`: simple single-process:: + 1. `~tornado.tcpserver.TCPServer.listen`: simple single-process:: server = HTTPServer(app) server.listen(8888) @@ -114,7 +114,7 @@ class HTTPServer(TCPServer): In many cases, `tornado.web.Application.listen` can be used to avoid the need to explicitly create the `HTTPServer`. - 2. `~tornado.netutil.TCPServer.bind`/`~tornado.netutil.TCPServer.start`: + 2. `~tornado.tcpserver.TCPServer.bind`/`~tornado.netutil.TCPServer.start`: simple multi-process:: server = HTTPServer(app) @@ -126,7 +126,7 @@ class HTTPServer(TCPServer): to the `HTTPServer` constructor. `start` will always start the server on the default singleton `IOLoop`. - 3. `~tornado.netutil.TCPServer.add_sockets`: advanced multi-process:: + 3. `~tornado.tcpserver.TCPServer.add_sockets`: advanced multi-process:: sockets = tornado.netutil.bind_sockets(8888) tornado.process.fork_processes(0) diff --git a/tornado/netutil.py b/tornado/netutil.py index 5247df008..4dc82d250 100644 --- a/tornado/netutil.py +++ b/tornado/netutil.py @@ -21,219 +21,13 @@ from __future__ import absolute_import, division, print_function, with_statement import errno import os import socket -import ssl import stat -from tornado import process from tornado.concurrent import dummy_executor, run_on_executor from tornado.ioloop import IOLoop -from tornado.iostream import IOStream, SSLIOStream -from tornado.log import app_log from tornado.platform.auto import set_close_exec -class TCPServer(object): - r"""A non-blocking, single-threaded TCP server. - - To use `TCPServer`, define a subclass which overrides the `handle_stream` - method. - - `TCPServer` can serve SSL traffic with Python 2.6+ and OpenSSL. - To make this server serve SSL traffic, send the ssl_options dictionary - argument with the arguments required for the `ssl.wrap_socket` method, - including "certfile" and "keyfile":: - - TCPServer(ssl_options={ - "certfile": os.path.join(data_dir, "mydomain.crt"), - "keyfile": os.path.join(data_dir, "mydomain.key"), - }) - - `TCPServer` initialization follows one of three patterns: - - 1. `listen`: simple single-process:: - - server = TCPServer() - server.listen(8888) - IOLoop.instance().start() - - 2. `bind`/`start`: simple multi-process:: - - server = TCPServer() - server.bind(8888) - server.start(0) # Forks multiple sub-processes - IOLoop.instance().start() - - When using this interface, an `IOLoop` must *not* be passed - to the `TCPServer` constructor. `start` will always start - the server on the default singleton `IOLoop`. - - 3. `add_sockets`: advanced multi-process:: - - sockets = bind_sockets(8888) - tornado.process.fork_processes(0) - server = TCPServer() - server.add_sockets(sockets) - IOLoop.instance().start() - - The `add_sockets` interface is more complicated, but it can be - used with `tornado.process.fork_processes` to give you more - flexibility in when the fork happens. `add_sockets` can - also be used in single-process servers if you want to create - your listening sockets in some way other than - `bind_sockets`. - """ - def __init__(self, io_loop=None, ssl_options=None): - self.io_loop = io_loop - self.ssl_options = ssl_options - self._sockets = {} # fd -> socket object - self._pending_sockets = [] - self._started = False - - # Verify the SSL options. Otherwise we don't get errors until clients - # connect. This doesn't verify that the keys are legitimate, but - # the SSL module doesn't do that until there is a connected socket - # which seems like too much work - if self.ssl_options is not None: - # Only certfile is required: it can contain both keys - if 'certfile' not in self.ssl_options: - raise KeyError('missing key "certfile" in ssl_options') - - if not os.path.exists(self.ssl_options['certfile']): - raise ValueError('certfile "%s" does not exist' % - self.ssl_options['certfile']) - if ('keyfile' in self.ssl_options and - not os.path.exists(self.ssl_options['keyfile'])): - raise ValueError('keyfile "%s" does not exist' % - self.ssl_options['keyfile']) - - def listen(self, port, address=""): - """Starts accepting connections on the given port. - - This method may be called more than once to listen on multiple ports. - `listen` takes effect immediately; it is not necessary to call - `TCPServer.start` afterwards. It is, however, necessary to start - the `IOLoop`. - """ - sockets = bind_sockets(port, address=address) - self.add_sockets(sockets) - - def add_sockets(self, sockets): - """Makes this server start accepting connections on the given sockets. - - The ``sockets`` parameter is a list of socket objects such as - those returned by `bind_sockets`. - `add_sockets` is typically used in combination with that - method and `tornado.process.fork_processes` to provide greater - control over the initialization of a multi-process server. - """ - if self.io_loop is None: - self.io_loop = IOLoop.instance() - - for sock in sockets: - self._sockets[sock.fileno()] = sock - add_accept_handler(sock, self._handle_connection, - io_loop=self.io_loop) - - def add_socket(self, socket): - """Singular version of `add_sockets`. Takes a single socket object.""" - self.add_sockets([socket]) - - def bind(self, port, address=None, family=socket.AF_UNSPEC, backlog=128): - """Binds this server to the given port on the given address. - - To start the server, call `start`. If you want to run this server - in a single process, you can call `listen` as a shortcut to the - sequence of `bind` and `start` calls. - - Address may be either an IP address or hostname. If it's a hostname, - the server will listen on all IP addresses associated with the - name. Address may be an empty string or None to listen on all - available interfaces. Family may be set to either ``socket.AF_INET`` - or ``socket.AF_INET6`` to restrict to ipv4 or ipv6 addresses, otherwise - both will be used if available. - - The ``backlog`` argument has the same meaning as for - `socket.listen`. - - This method may be called multiple times prior to `start` to listen - on multiple ports or interfaces. - """ - sockets = bind_sockets(port, address=address, family=family, - backlog=backlog) - if self._started: - self.add_sockets(sockets) - else: - self._pending_sockets.extend(sockets) - - def start(self, num_processes=1): - """Starts this server in the IOLoop. - - By default, we run the server in this process and do not fork any - additional child process. - - If num_processes is ``None`` or <= 0, we detect the number of cores - available on this machine and fork that number of child - processes. If num_processes is given and > 1, we fork that - specific number of sub-processes. - - Since we use processes and not threads, there is no shared memory - between any server code. - - Note that multiple processes are not compatible with the autoreload - module (or the ``debug=True`` option to `tornado.web.Application`). - When using multiple processes, no IOLoops can be created or - referenced until after the call to ``TCPServer.start(n)``. - """ - assert not self._started - self._started = True - if num_processes != 1: - process.fork_processes(num_processes) - sockets = self._pending_sockets - self._pending_sockets = [] - self.add_sockets(sockets) - - def stop(self): - """Stops listening for new connections. - - Requests currently in progress may still continue after the - server is stopped. - """ - for fd, sock in self._sockets.items(): - self.io_loop.remove_handler(fd) - sock.close() - - def handle_stream(self, stream, address): - """Override to handle a new `IOStream` from an incoming connection.""" - raise NotImplementedError() - - def _handle_connection(self, connection, address): - if self.ssl_options is not None: - assert ssl, "Python 2.6+ and OpenSSL required for SSL" - try: - connection = ssl.wrap_socket(connection, - server_side=True, - do_handshake_on_connect=False, - **self.ssl_options) - except ssl.SSLError as err: - if err.args[0] == ssl.SSL_ERROR_EOF: - return connection.close() - else: - raise - except socket.error as err: - if err.args[0] == errno.ECONNABORTED: - return connection.close() - else: - raise - try: - if self.ssl_options is not None: - stream = SSLIOStream(connection, io_loop=self.io_loop) - else: - stream = IOStream(connection, io_loop=self.io_loop) - self.handle_stream(stream, address) - except Exception: - app_log.error("Error in connection callback", exc_info=True) - - def bind_sockets(port, address=None, family=socket.AF_UNSPEC, backlog=128, flags=None): """Creates listening sockets bound to the given port and address. diff --git a/tornado/tcpserver.py b/tornado/tcpserver.py new file mode 100644 index 000000000..af30aa256 --- /dev/null +++ b/tornado/tcpserver.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python +# +# Copyright 2011 Facebook +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""A non-blocking, single-threaded TCP server.""" +from __future__ import absolute_import, division, print_function, with_statement + +import errno +import os +import socket +import ssl + +from tornado.log import app_log +from tornado.ioloop import IOLoop +from tornado.iostream import IOStream, SSLIOStream +from tornado.netutil import bind_sockets, add_accept_handler +from tornado import process + +class TCPServer(object): + r"""A non-blocking, single-threaded TCP server. + + To use `TCPServer`, define a subclass which overrides the `handle_stream` + method. + + `TCPServer` can serve SSL traffic with Python 2.6+ and OpenSSL. + To make this server serve SSL traffic, send the ssl_options dictionary + argument with the arguments required for the `ssl.wrap_socket` method, + including "certfile" and "keyfile":: + + TCPServer(ssl_options={ + "certfile": os.path.join(data_dir, "mydomain.crt"), + "keyfile": os.path.join(data_dir, "mydomain.key"), + }) + + `TCPServer` initialization follows one of three patterns: + + 1. `listen`: simple single-process:: + + server = TCPServer() + server.listen(8888) + IOLoop.instance().start() + + 2. `bind`/`start`: simple multi-process:: + + server = TCPServer() + server.bind(8888) + server.start(0) # Forks multiple sub-processes + IOLoop.instance().start() + + When using this interface, an `IOLoop` must *not* be passed + to the `TCPServer` constructor. `start` will always start + the server on the default singleton `IOLoop`. + + 3. `add_sockets`: advanced multi-process:: + + sockets = bind_sockets(8888) + tornado.process.fork_processes(0) + server = TCPServer() + server.add_sockets(sockets) + IOLoop.instance().start() + + The `add_sockets` interface is more complicated, but it can be + used with `tornado.process.fork_processes` to give you more + flexibility in when the fork happens. `add_sockets` can + also be used in single-process servers if you want to create + your listening sockets in some way other than + `bind_sockets`. + """ + def __init__(self, io_loop=None, ssl_options=None): + self.io_loop = io_loop + self.ssl_options = ssl_options + self._sockets = {} # fd -> socket object + self._pending_sockets = [] + self._started = False + + # Verify the SSL options. Otherwise we don't get errors until clients + # connect. This doesn't verify that the keys are legitimate, but + # the SSL module doesn't do that until there is a connected socket + # which seems like too much work + if self.ssl_options is not None: + # Only certfile is required: it can contain both keys + if 'certfile' not in self.ssl_options: + raise KeyError('missing key "certfile" in ssl_options') + + if not os.path.exists(self.ssl_options['certfile']): + raise ValueError('certfile "%s" does not exist' % + self.ssl_options['certfile']) + if ('keyfile' in self.ssl_options and + not os.path.exists(self.ssl_options['keyfile'])): + raise ValueError('keyfile "%s" does not exist' % + self.ssl_options['keyfile']) + + def listen(self, port, address=""): + """Starts accepting connections on the given port. + + This method may be called more than once to listen on multiple ports. + `listen` takes effect immediately; it is not necessary to call + `TCPServer.start` afterwards. It is, however, necessary to start + the `IOLoop`. + """ + sockets = bind_sockets(port, address=address) + self.add_sockets(sockets) + + def add_sockets(self, sockets): + """Makes this server start accepting connections on the given sockets. + + The ``sockets`` parameter is a list of socket objects such as + those returned by `bind_sockets`. + `add_sockets` is typically used in combination with that + method and `tornado.process.fork_processes` to provide greater + control over the initialization of a multi-process server. + """ + if self.io_loop is None: + self.io_loop = IOLoop.instance() + + for sock in sockets: + self._sockets[sock.fileno()] = sock + add_accept_handler(sock, self._handle_connection, + io_loop=self.io_loop) + + def add_socket(self, socket): + """Singular version of `add_sockets`. Takes a single socket object.""" + self.add_sockets([socket]) + + def bind(self, port, address=None, family=socket.AF_UNSPEC, backlog=128): + """Binds this server to the given port on the given address. + + To start the server, call `start`. If you want to run this server + in a single process, you can call `listen` as a shortcut to the + sequence of `bind` and `start` calls. + + Address may be either an IP address or hostname. If it's a hostname, + the server will listen on all IP addresses associated with the + name. Address may be an empty string or None to listen on all + available interfaces. Family may be set to either ``socket.AF_INET`` + or ``socket.AF_INET6`` to restrict to ipv4 or ipv6 addresses, otherwise + both will be used if available. + + The ``backlog`` argument has the same meaning as for + `socket.listen`. + + This method may be called multiple times prior to `start` to listen + on multiple ports or interfaces. + """ + sockets = bind_sockets(port, address=address, family=family, + backlog=backlog) + if self._started: + self.add_sockets(sockets) + else: + self._pending_sockets.extend(sockets) + + def start(self, num_processes=1): + """Starts this server in the IOLoop. + + By default, we run the server in this process and do not fork any + additional child process. + + If num_processes is ``None`` or <= 0, we detect the number of cores + available on this machine and fork that number of child + processes. If num_processes is given and > 1, we fork that + specific number of sub-processes. + + Since we use processes and not threads, there is no shared memory + between any server code. + + Note that multiple processes are not compatible with the autoreload + module (or the ``debug=True`` option to `tornado.web.Application`). + When using multiple processes, no IOLoops can be created or + referenced until after the call to ``TCPServer.start(n)``. + """ + assert not self._started + self._started = True + if num_processes != 1: + process.fork_processes(num_processes) + sockets = self._pending_sockets + self._pending_sockets = [] + self.add_sockets(sockets) + + def stop(self): + """Stops listening for new connections. + + Requests currently in progress may still continue after the + server is stopped. + """ + for fd, sock in self._sockets.items(): + self.io_loop.remove_handler(fd) + sock.close() + + def handle_stream(self, stream, address): + """Override to handle a new `IOStream` from an incoming connection.""" + raise NotImplementedError() + + def _handle_connection(self, connection, address): + if self.ssl_options is not None: + assert ssl, "Python 2.6+ and OpenSSL required for SSL" + try: + connection = ssl.wrap_socket(connection, + server_side=True, + do_handshake_on_connect=False, + **self.ssl_options) + except ssl.SSLError as err: + if err.args[0] == ssl.SSL_ERROR_EOF: + return connection.close() + else: + raise + except socket.error as err: + if err.args[0] == errno.ECONNABORTED: + return connection.close() + else: + raise + try: + if self.ssl_options is not None: + stream = SSLIOStream(connection, io_loop=self.io_loop) + else: + stream = IOStream(connection, io_loop=self.io_loop) + self.handle_stream(stream, address) + except Exception: + app_log.error("Error in connection callback", exc_info=True) diff --git a/tornado/test/concurrent_test.py b/tornado/test/concurrent_test.py index 2a2f1a0dd..994150cb3 100644 --- a/tornado/test/concurrent_test.py +++ b/tornado/test/concurrent_test.py @@ -23,7 +23,7 @@ from tornado.concurrent import Future, future_wrap from tornado.escape import utf8, to_unicode from tornado import gen from tornado.iostream import IOStream -from tornado.netutil import TCPServer +from tornado.tcpserver import TCPServer from tornado.testing import AsyncTestCase, LogTrapTestCase, get_unused_port diff --git a/website/sphinx/releases/next.rst b/website/sphinx/releases/next.rst index 1380b0f4f..b15e97e2a 100644 --- a/website/sphinx/releases/next.rst +++ b/website/sphinx/releases/next.rst @@ -215,3 +215,4 @@ In progress * `tornado.httpserver.HTTPConnection` now has a `set_close_callback` method that should be used instead of reaching into its ``stream`` attribute. +* `tornado.netutil.TCPServer` has moved to its own module, `tornado.tcpserver`.