From: Ben Darnell Date: Sun, 25 May 2014 16:50:51 +0000 (-0400) Subject: Stream large response bodies from StaticFileHandler. X-Git-Tag: v4.0.0b1~38 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=c93e961fe413796cbd48e329094d3147c84cdf6d;p=thirdparty%2Ftornado.git Stream large response bodies from StaticFileHandler. The get_content interface is still synchronous so it's not a complete solution, but this will keep the server from buffering the whole file in memory while writing it out to the client. Return Futures from the wsgi interface, allowing @gen.coroutine to be used there in limited circumstances. --- diff --git a/tornado/http1connection.py b/tornado/http1connection.py index 1490267a1..9f474e1b6 100644 --- a/tornado/http1connection.py +++ b/tornado/http1connection.py @@ -332,20 +332,21 @@ class HTTP1Connection(httputil.HTTPConnection): for line in lines: if b'\n' in line: raise ValueError('Newline in header: ' + repr(line)) + future = None if self.stream.closed(): - self._write_future = Future() - self._write_future.set_exception(iostream.StreamClosedError()) + future = self._write_future = Future() + future.set_exception(iostream.StreamClosedError()) else: if callback is not None: self._write_callback = stack_context.wrap(callback) else: - self._write_future = Future() + future = self._write_future = Future() data = b"\r\n".join(lines) + b"\r\n\r\n" if chunk: data += self._format_chunk(chunk) self._pending_write = self.stream.write(data) self._pending_write.add_done_callback(self._on_write_complete) - return self._write_future + return future def _format_chunk(self, chunk): if self._expected_content_remaining is not None: diff --git a/tornado/test/web_test.py b/tornado/test/web_test.py index 5ca6e4219..64593d2a4 100644 --- a/tornado/test/web_test.py +++ b/tornado/test/web_test.py @@ -1160,6 +1160,11 @@ class CustomStaticFileTest(WebTestCase): return b'bar' raise Exception("unexpected path %r" % path) + def get_content_size(self): + if self.absolute_path == 'CustomStaticFileTest:foo.txt': + return 3 + raise Exception("unexpected path %r" % self.absolute_path) + def get_modified_time(self): return None diff --git a/tornado/web.py b/tornado/web.py index 8668e0b33..5a875fd96 100644 --- a/tornado/web.py +++ b/tornado/web.py @@ -2034,8 +2034,9 @@ class StaticFileHandler(RequestHandler): cls._static_hashes = {} def head(self, path): - self.get(path, include_body=False) + return self.get(path, include_body=False) + @gen.coroutine def get(self, path, include_body=True): # Set up our path instance variables. self.path = self.parse_url_path(path) @@ -2060,9 +2061,9 @@ class StaticFileHandler(RequestHandler): # the request will be treated as if the header didn't exist. request_range = httputil._parse_request_range(range_header) + size = self.get_content_size() if request_range: start, end = request_range - size = self.get_content_size() if (start is not None and start >= size) or end == 0: # As per RFC 2616 14.35.1, a range is not satisfiable only: if # the first requested byte is equal to or greater than the @@ -2086,7 +2087,17 @@ class StaticFileHandler(RequestHandler): self.set_header("Content-Range", httputil._get_content_range(start, end, size)) else: - start = end = size = None + start = end = None + + if start is not None and end is not None: + content_length = end - start + elif end is not None: + content_length = end + elif start is not None: + content_length = size - start + else: + content_length = size + self.set_header("Content-Length", content_length) if include_body: content = self.get_content(self.absolute_path, start, end) @@ -2094,20 +2105,9 @@ class StaticFileHandler(RequestHandler): content = [content] for chunk in content: self.write(chunk) + yield self.flush() else: assert self.request.method == "HEAD" - if start is not None and end is not None: - content_length = end - start - elif end is not None: - content_length = end - else: - if size is None: - size = self.get_content_size() - if start is not None: - content_length = size - start - else: - content_length = size - self.set_header("Content-Length", content_length) def compute_etag(self): """Sets the ``Etag`` header based on static url version. @@ -2287,11 +2287,13 @@ class StaticFileHandler(RequestHandler): def get_content_size(self): """Retrieve the total size of the resource at the given path. - This method may be overridden by subclasses. It will only - be called if a partial result is requested from `get_content`, - or on ``HEAD`` requests. + This method may be overridden by subclasses. .. versionadded:: 3.1 + + .. versionchanged:: 3.3 + This method is now always called, instead of only when + partial results are requested. """ stat_result = self._stat() return stat_result[stat.ST_SIZE] diff --git a/tornado/wsgi.py b/tornado/wsgi.py index 0052d674c..47a0590a7 100644 --- a/tornado/wsgi.py +++ b/tornado/wsgi.py @@ -34,6 +34,7 @@ from __future__ import absolute_import, division, print_function, with_statement import sys import tornado +from tornado.concurrent import Future from tornado import escape from tornado import httputil from tornado.log import access_log @@ -84,6 +85,12 @@ class WSGIApplication(web.Application): return WSGIAdapter(self)(environ, start_response) +# WSGI has no facilities for flow control, so just return an already-done +# Future when the interface requires it. +_dummy_future = Future() +_dummy_future.set_result(None) + + class _WSGIConnection(httputil.HTTPConnection): def __init__(self, method, start_response, context): self.method = method @@ -113,6 +120,7 @@ class _WSGIConnection(httputil.HTTPConnection): self.write(chunk, callback) elif callback is not None: callback() + return _dummy_future def write(self, chunk, callback=None): if self._expected_content_remaining is not None: @@ -124,6 +132,7 @@ class _WSGIConnection(httputil.HTTPConnection): self._write_buffer.append(chunk) if callback is not None: callback() + return _dummy_future def finish(self): if (self._expected_content_remaining is not None and