From: A. Jesse Jiryu Davis Date: Sat, 5 Nov 2016 01:42:01 +0000 (-0400) Subject: docs: Demonstrate uploading and receiving files X-Git-Tag: v4.5.0~58^2 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=refs%2Fpull%2F1881%2Fhead;p=thirdparty%2Ftornado.git docs: Demonstrate uploading and receiving files --- diff --git a/demos/file_upload/file_receiver.py b/demos/file_upload/file_receiver.py new file mode 100644 index 000000000..3b3e98673 --- /dev/null +++ b/demos/file_upload/file_receiver.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python + +"""Usage: python file_receiver.py + +Demonstrates a server that receives a multipart-form-encoded set of files in an +HTTP POST, or streams in the raw data of a single file in an HTTP PUT. + +See file_uploader.py in this directory for code that uploads files in this format. +""" + +import logging + +try: + from urllib.parse import unquote +except ImportError: + # Python 2. + from urllib import unquote + +import tornado.ioloop +import tornado.web +from tornado import options + + +class POSTHandler(tornado.web.RequestHandler): + def post(self): + for field_name, files in self.request.files.items(): + for info in files: + filename, content_type = info['filename'], info['content_type'] + body = info['body'] + logging.info('POST "%s" "%s" %d bytes', + filename, content_type, len(body)) + + self.write('OK') + + +@tornado.web.stream_request_body +class PUTHandler(tornado.web.RequestHandler): + def initialize(self): + self.bytes_read = 0 + + def data_received(self, chunk): + self.bytes_read += len(chunk) + + def put(self, filename): + filename = unquote(filename) + mtype = self.request.headers.get('Content-Type') + logging.info('PUT "%s" "%s" %d bytes', filename, mtype, self.bytes_read) + self.write('OK') + + +def make_app(): + return tornado.web.Application([ + (r"/post", POSTHandler), + (r"/(.*)", PUTHandler), + ]) + + +if __name__ == "__main__": + # Tornado configures logging. + options.parse_command_line() + app = make_app() + app.listen(8888) + tornado.ioloop.IOLoop.current().start() diff --git a/demos/file_upload/file_uploader.py b/demos/file_upload/file_uploader.py new file mode 100644 index 000000000..025c2159e --- /dev/null +++ b/demos/file_upload/file_uploader.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python + +"""Usage: python file_uploader.py [--put] file1.txt file2.png ... + +Demonstrates uploading files to a server, without concurrency. It can either +POST a multipart-form-encoded request containing one or more files, or PUT a +single file without encoding. + +See also file_receiver.py in this directory, a server that receives uploads. +""" + +import mimetypes +import os +import sys +from functools import partial +from uuid import uuid4 + +try: + from urllib.parse import quote +except ImportError: + # Python 2. + from urllib import quote + +from tornado import gen, httpclient, ioloop +from tornado.options import define, options + + +# Using HTTP POST, upload one or more files in a single multipart-form-encoded +# request. +@gen.coroutine +def multipart_producer(boundary, filenames, write): + boundary_bytes = boundary.encode() + + for filename in filenames: + filename_bytes = filename.encode() + write(b'--%s\r\n' % (boundary_bytes,)) + write(b'Content-Disposition: form-data; name="%s"; filename="%s"\r\n' % + (filename_bytes, filename_bytes)) + + mtype = mimetypes.guess_type(filename)[0] or 'application/octet-stream' + write(b'Content-Type: %s\r\n' % (mtype.encode(),)) + write(b'\r\n') + with open(filename, 'rb') as f: + while True: + # 16k at a time. + chunk = f.read(16 * 1024) + if not chunk: + break + write(chunk) + + # Let the IOLoop process its event queue. + yield gen.moment + + write(b'\r\n') + yield gen.moment + + write(b'--%s--\r\n' % (boundary_bytes,)) + + +# Using HTTP PUT, upload one raw file. This is preferred for large files since +# the server can stream the data instead of buffering it entirely in memory. +@gen.coroutine +def post(filenames): + client = httpclient.AsyncHTTPClient() + boundary = uuid4().hex + headers = {'Content-Type': 'multipart/form-data; boundary=%s' % boundary} + producer = partial(multipart_producer, boundary, filenames) + response = yield client.fetch('http://localhost:8888/post', + method='POST', + headers=headers, + body_producer=producer) + + print(response) + + +@gen.coroutine +def raw_producer(filename, write): + with open(filename, 'rb') as f: + while True: + # 16K at a time. + chunk = f.read(16 * 1024) + if not chunk: + # Complete. + break + + write(chunk) + + +@gen.coroutine +def put(filenames): + client = httpclient.AsyncHTTPClient() + for filename in filenames: + mtype = mimetypes.guess_type(filename)[0] or 'application/octet-stream' + headers = {'Content-Type': mtype} + producer = partial(raw_producer, filename) + url_path = quote(os.path.basename(filename)) + response = yield client.fetch('http://localhost:8888/%s' % url_path, + method='PUT', + headers=headers, + body_producer=producer) + + print(response) + + +define("put", type=bool, help="Use PUT instead of POST", group="file uploader") + +# Tornado configures logging from command line opts and returns remaining args. +filenames = options.parse_command_line() +if not filenames: + print("Provide a list of filenames to upload.", file=sys.stderr) + sys.exit(1) + +method = put if options.put else post +ioloop.IOLoop.current().run_sync(lambda: method(filenames)) diff --git a/docs/guide/structure.rst b/docs/guide/structure.rst index f0829df0a..071c50a27 100644 --- a/docs/guide/structure.rst +++ b/docs/guide/structure.rst @@ -153,6 +153,10 @@ By default uploaded files are fully buffered in memory; if you need to handle files that are too large to comfortably keep in memory see the `.stream_request_body` class decorator. +In the demos directory, +`file_receiver.py `_ +shows both methods of receiving file uploads. + Due to the quirks of the HTML form encoding (e.g. the ambiguity around singular versus plural arguments), Tornado does not attempt to unify form arguments with other types of input. In particular, we do not diff --git a/docs/httpclient.rst b/docs/httpclient.rst index a641fa293..53a0a8812 100644 --- a/docs/httpclient.rst +++ b/docs/httpclient.rst @@ -50,3 +50,11 @@ Implementations .. class:: CurlAsyncHTTPClient(io_loop, max_clients=10, defaults=None) ``libcurl``-based HTTP client. + +Example Code +~~~~~~~~~~~~ + +* `A simple webspider `_ + shows how to fetch URLs concurrently. +* `The file uploader demo `_ + uses either HTTP POST or HTTP PUT to upload files to a server. diff --git a/tornado/web.py b/tornado/web.py index a0cb0e8eb..f939673e3 100644 --- a/tornado/web.py +++ b/tornado/web.py @@ -1670,6 +1670,9 @@ def stream_request_body(cls): There is a subtle interaction between ``data_received`` and asynchronous ``prepare``: The first call to ``data_received`` may occur at any point after the call to ``prepare`` has returned *or yielded*. + + See the `file receiver demo `_ + for example usage. """ if not issubclass(cls, RequestHandler): raise TypeError("expected subclass of RequestHandler, got %r", cls)