From: Ben Darnell Date: Sun, 8 Apr 2018 15:39:14 +0000 (-0400) Subject: testing: Add raise_error argument to AsyncHTTPTestCase.fetch X-Git-Tag: v5.1.0b1~30^2~4 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=25294e405fc134ad76982961af6c8a53e4435af3;p=thirdparty%2Ftornado.git testing: Add raise_error argument to AsyncHTTPTestCase.fetch Give it the same deprecation behavior as AsyncHTTPClient.fetch --- diff --git a/tornado/test/curl_httpclient_test.py b/tornado/test/curl_httpclient_test.py index cc29ee10b..d0cfa979b 100644 --- a/tornado/test/curl_httpclient_test.py +++ b/tornado/test/curl_httpclient_test.py @@ -4,12 +4,12 @@ from __future__ import absolute_import, division, print_function from hashlib import md5 from tornado.escape import utf8 -from tornado.httpclient import HTTPRequest +from tornado.httpclient import HTTPRequest, HTTPClientError from tornado.locks import Event from tornado.stack_context import ExceptionStackContext from tornado.testing import AsyncHTTPTestCase, gen_test from tornado.test import httpclient_test -from tornado.test.util import unittest +from tornado.test.util import unittest, ignore_deprecation from tornado.web import Application, RequestHandler @@ -131,5 +131,14 @@ class CurlHTTPClientTestCase(AsyncHTTPTestCase): def test_failed_setup(self): self.http_client = self.create_client(max_clients=1) for i in range(5): - response = self.fetch(u'/ユニコード') + with ignore_deprecation(): + response = self.fetch(u'/ユニコード') self.assertIsNot(response.error, None) + + with self.assertRaises((UnicodeEncodeError, HTTPClientError)): + # This raises UnicodeDecodeError on py3 and + # HTTPClientError(404) on py2. The main motivation of + # this test is to ensure that the UnicodeEncodeError + # during the setup phase doesn't lead the request to + # be dropped on the floor. + response = self.fetch(u'/ユニコード', raise_error=True) diff --git a/tornado/test/httpclient_test.py b/tornado/test/httpclient_test.py index e135526f5..f980b48cd 100644 --- a/tornado/test/httpclient_test.py +++ b/tornado/test/httpclient_test.py @@ -254,10 +254,10 @@ Transfer-Encoding: chunked # on an unknown mode. with ExpectLog(gen_log, "uncaught exception", required=False): with self.assertRaises((ValueError, HTTPError)): - response = self.fetch("/auth", auth_username="Aladdin", - auth_password="open sesame", - auth_mode="asdf") - response.rethrow() + self.fetch("/auth", auth_username="Aladdin", + auth_password="open sesame", + auth_mode="asdf", + raise_error=True) def test_follow_redirect(self): response = self.fetch("/countdown/2", follow_redirects=False) @@ -481,8 +481,7 @@ X-XSS-Protection: 1; # These methods require a body. for method in ('POST', 'PUT', 'PATCH'): with self.assertRaises(ValueError) as context: - resp = self.fetch('/all_methods', method=method) - resp.rethrow() + self.fetch('/all_methods', method=method, raise_error=True) self.assertIn('must not be None', str(context.exception)) resp = self.fetch('/all_methods', method=method, @@ -492,16 +491,14 @@ X-XSS-Protection: 1; # These methods don't allow a body. for method in ('GET', 'DELETE', 'OPTIONS'): with self.assertRaises(ValueError) as context: - resp = self.fetch('/all_methods', method=method, body=b'asdf') - resp.rethrow() + self.fetch('/all_methods', method=method, body=b'asdf', raise_error=True) self.assertIn('must be None', str(context.exception)) # In most cases this can be overridden, but curl_httpclient # does not allow body with a GET at all. if method != 'GET': - resp = self.fetch('/all_methods', method=method, body=b'asdf', - allow_nonstandard_methods=True) - resp.rethrow() + self.fetch('/all_methods', method=method, body=b'asdf', + allow_nonstandard_methods=True, raise_error=True) self.assertEqual(resp.code, 200) # This test causes odd failures with the combination of diff --git a/tornado/test/httpserver_test.py b/tornado/test/httpserver_test.py index 341dfbe8f..b53dd8bfd 100644 --- a/tornado/test/httpserver_test.py +++ b/tornado/test/httpserver_test.py @@ -4,6 +4,7 @@ from tornado import netutil from tornado.concurrent import Future from tornado.escape import json_decode, json_encode, utf8, _unicode, recursive_unicode, native_str from tornado.http1connection import HTTP1Connection +from tornado.httpclient import HTTPError from tornado.httpserver import HTTPServer from tornado.httputil import HTTPHeaders, HTTPMessageDelegate, HTTPServerConnectionDelegate, ResponseStartLine # noqa: E501 from tornado.iostream import IOStream @@ -109,18 +110,19 @@ class SSLTestMixin(object): # misbehaving. with ExpectLog(gen_log, '(SSL Error|uncaught exception)'): with ExpectLog(gen_log, 'Uncaught exception', required=False): - response = self.fetch( - self.get_url("/").replace('https:', 'http:'), - request_timeout=3600, - connect_timeout=3600) - self.assertEqual(response.code, 599) + with self.assertRaises((IOError, HTTPError)): + self.fetch( + self.get_url("/").replace('https:', 'http:'), + request_timeout=3600, + connect_timeout=3600, + raise_error=True) def test_error_logging(self): # No stack traces are logged for SSL errors. with ExpectLog(gen_log, 'SSL Error') as expect_log: - response = self.fetch( - self.get_url("/").replace("https:", "http:")) - self.assertEqual(response.code, 599) + with self.assertRaises((IOError, HTTPError)): + self.fetch(self.get_url("/").replace("https:", "http:"), + raise_error=True) self.assertFalse(expect_log.logged_stack) # Python's SSL implementation differs significantly between versions. @@ -961,11 +963,14 @@ class MaxHeaderSizeTest(AsyncHTTPTestCase): def test_large_headers(self): with ExpectLog(gen_log, "Unsatisfiable read", required=False): - response = self.fetch("/", headers={'X-Filler': 'a' * 1000}) - # 431 is "Request Header Fields Too Large", defined in RFC - # 6585. However, many implementations just close the - # connection in this case, resulting in a 599. - self.assertIn(response.code, (431, 599)) + try: + self.fetch("/", headers={'X-Filler': 'a' * 1000}, raise_error=True) + self.fail("did not raise expected exception") + except HTTPError as e: + # 431 is "Request Header Fields Too Large", defined in RFC + # 6585. However, many implementations just close the + # connection in this case, resulting in a 599. + self.assertIn(e.response.code, (431, 599)) @skipOnTravis @@ -1062,24 +1067,25 @@ class BodyLimitsTest(AsyncHTTPTestCase): response = self.fetch('/buffered', method='PUT', body=b'a' * 10240) self.assertEqual(response.code, 400) + @unittest.skipIf(os.name == 'nt', 'flaky on windows') def test_large_body_buffered_chunked(self): + # This test is flaky on windows for unknown reasons. with ExpectLog(gen_log, '.*chunked body too large'): response = self.fetch('/buffered', method='PUT', body_producer=lambda write: write(b'a' * 10240)) - # this test is flaky on windows; accept 400 (expected) or 599 - self.assertIn(response.code, [400, 599]) + self.assertEqual(response.code, 400) def test_large_body_streaming(self): with ExpectLog(gen_log, '.*Content-Length too long'): response = self.fetch('/streaming', method='PUT', body=b'a' * 10240) self.assertEqual(response.code, 400) + @unittest.skipIf(os.name == 'nt', 'flaky on windows') def test_large_body_streaming_chunked(self): with ExpectLog(gen_log, '.*chunked body too large'): response = self.fetch('/streaming', method='PUT', body_producer=lambda write: write(b'a' * 10240)) - # this test is flaky on windows; accept 400 (expected) or 599 - self.assertIn(response.code, [400, 599]) + self.assertEqual(response.code, 400) def test_large_body_streaming_override(self): response = self.fetch('/streaming?expected_size=10240', method='PUT', diff --git a/tornado/test/simple_httpclient_test.py b/tornado/test/simple_httpclient_test.py index a8893a99e..96fbff0ee 100644 --- a/tornado/test/simple_httpclient_test.py +++ b/tornado/test/simple_httpclient_test.py @@ -13,17 +13,20 @@ import sys from tornado.escape import to_unicode from tornado import gen -from tornado.httpclient import AsyncHTTPClient +from tornado.httpclient import AsyncHTTPClient, HTTPError from tornado.httputil import HTTPHeaders, ResponseStartLine from tornado.ioloop import IOLoop +from tornado.iostream import UnsatisfiableReadError from tornado.log import gen_log from tornado.concurrent import Future from tornado.netutil import Resolver, bind_sockets from tornado.simple_httpclient import SimpleAsyncHTTPClient from tornado.test.httpclient_test import ChunkHandler, CountdownHandler, HelloWorldHandler, RedirectHandler # noqa: E501 from tornado.test import httpclient_test -from tornado.testing import AsyncHTTPTestCase, AsyncHTTPSTestCase, AsyncTestCase, ExpectLog, gen_test -from tornado.test.util import skipOnTravis, skipIfNoIPv6, refusing_port, skipBefore35, exec_test, ignore_deprecation +from tornado.testing import (AsyncHTTPTestCase, AsyncHTTPSTestCase, AsyncTestCase, + ExpectLog, gen_test) +from tornado.test.util import (skipOnTravis, skipIfNoIPv6, refusing_port, skipBefore35, + exec_test, ignore_deprecation) from tornado.web import RequestHandler, Application, asynchronous, url, stream_request_body @@ -263,8 +266,9 @@ class SimpleHTTPClientTestMixin(object): timeout = 0.5 timeout_min, timeout_max = 0.4, 0.6 - response = self.fetch('/trigger?wake=false', request_timeout=timeout) - self.assertEqual(response.code, 599) + with ignore_deprecation(): + response = self.fetch('/trigger?wake=false', request_timeout=timeout) + self.assertEqual(response.code, 599) self.assertTrue(timeout_min < response.request_time < timeout_max, response.request_time) self.assertEqual(str(response.error), "HTTP 599: Timeout during request") @@ -279,8 +283,8 @@ class SimpleHTTPClientTestMixin(object): url = '%s://[::1]:%d/hello' % (self.get_protocol(), port) # ipv6 is currently enabled by default but can be disabled - response = self.fetch(url, allow_ipv6=False) - self.assertEqual(response.code, 599) + with self.assertRaises(Exception): + self.fetch(url, allow_ipv6=False, raise_error=True) response = self.fetch(url) self.assertEqual(response.body, b"Hello world!") @@ -331,7 +335,8 @@ class SimpleHTTPClientTestMixin(object): cleanup_func, port = refusing_port() self.addCleanup(cleanup_func) with ExpectLog(gen_log, ".*", required=False): - response = self.fetch("http://127.0.0.1:%d/" % port) + with ignore_deprecation(): + response = self.fetch("http://127.0.0.1:%d/" % port) self.assertEqual(599, response.code) if sys.platform != 'cygwin': @@ -502,25 +507,25 @@ class SimpleHTTPSClientTestCase(SimpleHTTPClientTestMixin, AsyncHTTPSTestCase): def test_ssl_options_handshake_fail(self): with ExpectLog(gen_log, "SSL Error|Uncaught exception", required=False): - resp = self.fetch( - "/hello", ssl_options=dict(cert_reqs=ssl.CERT_REQUIRED)) - self.assertRaises(ssl.SSLError, resp.rethrow) + with self.assertRaises(ssl.SSLError): + self.fetch( + "/hello", ssl_options=dict(cert_reqs=ssl.CERT_REQUIRED), + raise_error=True) def test_ssl_context_handshake_fail(self): with ExpectLog(gen_log, "SSL Error|Uncaught exception"): ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) ctx.verify_mode = ssl.CERT_REQUIRED - resp = self.fetch("/hello", ssl_options=ctx) - self.assertRaises(ssl.SSLError, resp.rethrow) + with self.assertRaises(ssl.SSLError): + self.fetch("/hello", ssl_options=ctx, raise_error=True) def test_error_logging(self): # No stack traces are logged for SSL errors (in this case, # failure to validate the testing self-signed cert). # The SSLError is exposed through ssl.SSLError. with ExpectLog(gen_log, '.*') as expect_log: - response = self.fetch("/", validate_cert=True) - self.assertEqual(response.code, 599) - self.assertIsInstance(response.error, ssl.SSLError) + with self.assertRaises(ssl.SSLError): + self.fetch("/", validate_cert=True, raise_error=True) self.assertFalse(expect_log.logged_stack) @@ -621,7 +626,8 @@ class HTTP204NoContentTestCase(AsyncHTTPTestCase): def test_204_invalid_content_length(self): # 204 status with non-zero content length is malformed with ExpectLog(gen_log, ".*Response with code 204 should not have body"): - response = self.fetch("/?error=1") + with ignore_deprecation(): + response = self.fetch("/?error=1") if not self.http1: self.skipTest("requires HTTP/1.x") if self.http_client.configured_class != SimpleAsyncHTTPClient: @@ -668,8 +674,8 @@ class ResolveTimeoutTestCase(AsyncHTTPTestCase): return Application([url("/hello", HelloWorldHandler), ]) def test_resolve_timeout(self): - response = self.fetch('/hello', connect_timeout=0.1) - self.assertEqual(response.code, 599) + with self.assertRaises(TypeError): + self.fetch('/hello', connect_timeout=0.1, raise_error=True) class MaxHeaderSizeTest(AsyncHTTPTestCase): @@ -697,8 +703,8 @@ class MaxHeaderSizeTest(AsyncHTTPTestCase): def test_large_headers(self): with ExpectLog(gen_log, "Unsatisfiable read"): - response = self.fetch('/large') - self.assertEqual(response.code, 599) + with self.assertRaises(UnsatisfiableReadError): + self.fetch('/large', raise_error=True) class MaxBodySizeTest(AsyncHTTPTestCase): @@ -724,8 +730,8 @@ class MaxBodySizeTest(AsyncHTTPTestCase): def test_large_body(self): with ExpectLog(gen_log, "Malformed HTTP message from None: Content-Length too long"): - response = self.fetch('/large') - self.assertEqual(response.code, 599) + with self.assertRaises(HTTPError): + self.fetch('/large', raise_error=True) class MaxBufferSizeTest(AsyncHTTPTestCase): @@ -765,5 +771,5 @@ class ChunkedWithContentLengthTest(AsyncHTTPTestCase): # Make sure the invalid headers are detected with ExpectLog(gen_log, ("Malformed HTTP message from None: Response " "with both Transfer-Encoding and Content-Length")): - response = self.fetch('/chunkwithcl') - self.assertEqual(response.code, 599) + with self.assertRaises(HTTPError): + self.fetch('/chunkwithcl', raise_error=True) diff --git a/tornado/test/testing_test.py b/tornado/test/testing_test.py index 374494668..eb3445e90 100644 --- a/tornado/test/testing_test.py +++ b/tornado/test/testing_test.py @@ -2,7 +2,7 @@ from __future__ import absolute_import, division, print_function from tornado import gen, ioloop from tornado.log import app_log -from tornado.test.util import unittest, skipBefore35, exec_test +from tornado.test.util import unittest, skipBefore35, exec_test, ignore_deprecation from tornado.testing import AsyncHTTPTestCase, AsyncTestCase, bind_unused_port, gen_test, ExpectLog from tornado.web import Application import contextlib @@ -99,13 +99,15 @@ class AsyncHTTPTestCaseTest(AsyncHTTPTestCase): def test_fetch_full_http_url(self): path = 'http://localhost:%d/path' % self.external_port - response = self.fetch(path, request_timeout=0.1) + with ignore_deprecation(): + response = self.fetch(path, request_timeout=0.1, raise_error=False) self.assertEqual(response.request.url, path) def test_fetch_full_https_url(self): path = 'https://localhost:%d/path' % self.external_port - response = self.fetch(path, request_timeout=0.1) + with ignore_deprecation(): + response = self.fetch(path, request_timeout=0.1) self.assertEqual(response.request.url, path) @classmethod diff --git a/tornado/test/web_test.py b/tornado/test/web_test.py index d757d78e7..cd9df37af 100644 --- a/tornado/test/web_test.py +++ b/tornado/test/web_test.py @@ -1,7 +1,7 @@ from __future__ import absolute_import, division, print_function from tornado.concurrent import Future -from tornado import gen +from tornado import gen, httpclient from tornado.escape import json_decode, utf8, to_unicode, recursive_unicode, native_str, to_basestring # noqa: E501 from tornado.httputil import format_timestamp from tornado.ioloop import IOLoop @@ -2332,8 +2332,8 @@ class IncorrectContentLengthTest(SimpleHandlerTestCase): with ExpectLog(gen_log, "(Cannot send error response after headers written" "|Failed to flush partial response)"): - response = self.fetch("/high") - self.assertEqual(response.code, 599) + with self.assertRaises(httpclient.HTTPError): + self.fetch("/high", raise_error=True) self.assertEqual(str(self.server_error), "Tried to write 40 bytes less than Content-Length") @@ -2345,8 +2345,8 @@ class IncorrectContentLengthTest(SimpleHandlerTestCase): with ExpectLog(gen_log, "(Cannot send error response after headers written" "|Failed to flush partial response)"): - response = self.fetch("/low") - self.assertEqual(response.code, 599) + with self.assertRaises(httpclient.HTTPError): + self.fetch("/low", raise_error=True) self.assertEqual(str(self.server_error), "Tried to write more data than Content-Length") @@ -2367,7 +2367,8 @@ class ClientCloseTest(SimpleHandlerTestCase): self.write('requires HTTP/1.x') def test_client_close(self): - response = self.fetch('/') + with ignore_deprecation(): + response = self.fetch('/') if response.body == b'requires HTTP/1.x': self.skipTest('requires HTTP/1.x') self.assertEqual(response.code, 599) diff --git a/tornado/testing.py b/tornado/testing.py index ab1779f20..04ea3816a 100644 --- a/tornado/testing.py +++ b/tornado/testing.py @@ -13,7 +13,7 @@ from __future__ import absolute_import, division, print_function try: from tornado import gen - from tornado.httpclient import AsyncHTTPClient, HTTPError, HTTPResponse + from tornado.httpclient import AsyncHTTPClient from tornado.httpserver import HTTPServer from tornado.simple_httpclient import SimpleAsyncHTTPClient from tornado.ioloop import IOLoop, TimeoutError @@ -80,6 +80,7 @@ else: import tornado.platform.asyncio _NON_OWNED_IOLOOPS = tornado.platform.asyncio.AsyncIOMainLoop + def bind_unused_port(reuse_port=False): """Binds a server socket to an available port on localhost. @@ -386,7 +387,7 @@ class AsyncHTTPTestCase(AsyncTestCase): """ raise NotImplementedError() - def fetch(self, path, **kwargs): + def fetch(self, path, raise_error=False, **kwargs): """Convenience method to synchronously fetch a URL. The given path will be appended to the local server's host and @@ -397,34 +398,36 @@ class AsyncHTTPTestCase(AsyncTestCase): If the path begins with http:// or https://, it will be treated as a full URL and will be fetched as-is. - Unlike awaiting `.AsyncHTTPClient.fetch` in a coroutine, no - exception is raised for non-200 response codes (as if the - ``raise_error=True`` option were used). + If ``raise_error`` is True, a `tornado.httpclient.HTTPError` will + be raised if the response code is not 200. This is the same behavior + as the ``raise_error`` argument to `.AsyncHTTPClient.fetch`, but + the default is False here (it's True in `.AsyncHTTPClient`) because + tests often need to deal with non-200 response codes. .. versionchanged:: 5.0 Added support for absolute URLs. + .. versionchanged:: 5.1 + + Added the ``raise_error`` argument. + .. deprecated:: 5.1 This method currently turns any exception into an `.HTTPResponse` with status code 599. In Tornado 6.0, - errors other than `tornado.httpclient.HTTPError` - will be passed through, and this method will only suppress - errors that would be raised due to non-200 response codes. + errors other than `tornado.httpclient.HTTPError` will be + passed through, and ``raise_error=False`` will only + suppress errors that would be raised due to non-200 + response codes. """ if path.lower().startswith(('http://', 'https://')): url = path else: url = self.get_url(path) - try: - return self.io_loop.run_sync(lambda: self.http_client.fetch(url, **kwargs)) - except HTTPError as e: - if e.response is not None: - return e.response - return HTTPResponse(None, 599, error=e, effective_url='unknown') - except Exception as e: - return HTTPResponse(None, 599, error=e, effective_url='unknown') + return self.io_loop.run_sync( + lambda: self.http_client.fetch(url, raise_error=raise_error, **kwargs), + timeout=get_async_test_timeout()) def get_httpserver_options(self): """May be overridden by subclasses to return additional