Stream large response bodies from StaticFileHandler.

The get_content interface is still synchronous so it's not a complete
solution, but this will keep the server from buffering the whole file in
memory while writing it out to the client.

Return Futures from the wsgi interface, allowing @gen.coroutine to be
used there in limited circumstances.

diff --git a/tornado/http1connection.py b/tornado/http1connection.py
index 1490267a10c307e4d4e633ed71fb6df41e878158..9f474e1b63fc1b22973e1a42fe74d2c0d1d39082 100644 (file)
--- a/tornado/http1connection.py
+++ b/tornado/http1connection.py
@@ -332,20 +332,21 @@ class HTTP1Connection(httputil.HTTPConnection):
         for line in lines:
             if b'\n' in line:
                 raise ValueError('Newline in header: ' + repr(line))
+        future = None
         if self.stream.closed():
-            self._write_future = Future()
-            self._write_future.set_exception(iostream.StreamClosedError())
+            future = self._write_future = Future()
+            future.set_exception(iostream.StreamClosedError())
         else:
             if callback is not None:
                 self._write_callback = stack_context.wrap(callback)
             else:
-                self._write_future = Future()
+                future = self._write_future = Future()
             data = b"\r\n".join(lines) + b"\r\n\r\n"
             if chunk:
                 data += self._format_chunk(chunk)
             self._pending_write = self.stream.write(data)
             self._pending_write.add_done_callback(self._on_write_complete)
-        return self._write_future
+        return future
 
     def _format_chunk(self, chunk):
         if self._expected_content_remaining is not None:

diff --git a/tornado/test/web_test.py b/tornado/test/web_test.py
index 5ca6e42196304bddeddff446286b4f66f207f5af..64593d2a495d35c410881ca94c689c9a52d34ead 100644 (file)
--- a/tornado/test/web_test.py
+++ b/tornado/test/web_test.py
@@ -1160,6 +1160,11 @@ class CustomStaticFileTest(WebTestCase):
                     return b'bar'
                 raise Exception("unexpected path %r" % path)
 
+            def get_content_size(self):
+                if self.absolute_path == 'CustomStaticFileTest:foo.txt':
+                    return 3
+                raise Exception("unexpected path %r" % self.absolute_path)
+
             def get_modified_time(self):
                 return None
 

diff --git a/tornado/web.py b/tornado/web.py
index 8668e0b336bd267640241ebf9924c8dc3b92fe51..5a875fd965531a4dceb07596a554b07db7a74d35 100644 (file)
--- a/tornado/web.py
+++ b/tornado/web.py
@@ -2034,8 +2034,9 @@ class StaticFileHandler(RequestHandler):
             cls._static_hashes = {}
 
     def head(self, path):
-        self.get(path, include_body=False)
+        return self.get(path, include_body=False)
 
+    @gen.coroutine
     def get(self, path, include_body=True):
         # Set up our path instance variables.
         self.path = self.parse_url_path(path)
@@ -2060,9 +2061,9 @@ class StaticFileHandler(RequestHandler):
             # the request will be treated as if the header didn't exist.
             request_range = httputil._parse_request_range(range_header)
 
+        size = self.get_content_size()
         if request_range:
             start, end = request_range
-            size = self.get_content_size()
             if (start is not None and start >= size) or end == 0:
                 # As per RFC 2616 14.35.1, a range is not satisfiable only: if
                 # the first requested byte is equal to or greater than the
@@ -2086,7 +2087,17 @@ class StaticFileHandler(RequestHandler):
                 self.set_header("Content-Range",
                                 httputil._get_content_range(start, end, size))
         else:
-            start = end = size = None
+            start = end = None
+
+        if start is not None and end is not None:
+            content_length = end - start
+        elif end is not None:
+            content_length = end
+        elif start is not None:
+            content_length = size - start
+        else:
+            content_length = size
+        self.set_header("Content-Length", content_length)
 
         if include_body:
             content = self.get_content(self.absolute_path, start, end)
@@ -2094,20 +2105,9 @@ class StaticFileHandler(RequestHandler):
                 content = [content]
             for chunk in content:
                 self.write(chunk)
+                yield self.flush()
         else:
             assert self.request.method == "HEAD"
-            if start is not None and end is not None:
-                content_length = end - start
-            elif end is not None:
-                content_length = end
-            else:
-                if size is None:
-                    size = self.get_content_size()
-                if start is not None:
-                    content_length = size - start
-                else:
-                    content_length = size
-            self.set_header("Content-Length", content_length)
 
     def compute_etag(self):
         """Sets the ``Etag`` header based on static url version.
@@ -2287,11 +2287,13 @@ class StaticFileHandler(RequestHandler):
     def get_content_size(self):
         """Retrieve the total size of the resource at the given path.
 
-        This method may be overridden by subclasses. It will only
-        be called if a partial result is requested from `get_content`,
-        or on ``HEAD`` requests.
+        This method may be overridden by subclasses.
 
         .. versionadded:: 3.1
+
+        .. versionchanged:: 3.3
+           This method is now always called, instead of only when
+           partial results are requested.
         """
         stat_result = self._stat()
         return stat_result[stat.ST_SIZE]

diff --git a/tornado/wsgi.py b/tornado/wsgi.py
index 0052d674cfbefe63fdd28d8dcae3efb3ed1c4bda..47a0590a799af814543650a5546ab279bdab4b03 100644 (file)
--- a/tornado/wsgi.py
+++ b/tornado/wsgi.py
@@ -34,6 +34,7 @@ from __future__ import absolute_import, division, print_function, with_statement
 import sys
 import tornado
 
+from tornado.concurrent import Future
 from tornado import escape
 from tornado import httputil
 from tornado.log import access_log
@@ -84,6 +85,12 @@ class WSGIApplication(web.Application):
         return WSGIAdapter(self)(environ, start_response)
 
 
+# WSGI has no facilities for flow control, so just return an already-done
+# Future when the interface requires it.
+_dummy_future = Future()
+_dummy_future.set_result(None)
+
+
 class _WSGIConnection(httputil.HTTPConnection):
     def __init__(self, method, start_response, context):
         self.method = method
@@ -113,6 +120,7 @@ class _WSGIConnection(httputil.HTTPConnection):
             self.write(chunk, callback)
         elif callback is not None:
             callback()
+        return _dummy_future
 
     def write(self, chunk, callback=None):
         if self._expected_content_remaining is not None:
@@ -124,6 +132,7 @@ class _WSGIConnection(httputil.HTTPConnection):
         self._write_buffer.append(chunk)
         if callback is not None:
             callback()
+        return _dummy_future
 
     def finish(self):
         if (self._expected_content_remaining is not None and