From: Tom Christie Date: Mon, 29 Apr 2019 10:54:48 +0000 (+0100) Subject: Restructuring X-Git-Tag: 0.3.0~66^2~7 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=e09fc3741f8b5f311a463634afcef0f3cfd97243;p=thirdparty%2Fhttpx.git Restructuring --- diff --git a/httpcore/__init__.py b/httpcore/__init__.py index ef8d6a5d..b81c8779 100644 --- a/httpcore/__init__.py +++ b/httpcore/__init__.py @@ -1,8 +1,10 @@ -from .adapters import Adapter +from .adapters.redirects import RedirectAdapter from .client import Client from .config import PoolLimits, SSLConfig, TimeoutConfig -from .connection import HTTPConnection -from .connection_pool import ConnectionPool +from .dispatch.connection import HTTPConnection +from .dispatch.connection_pool import ConnectionPool +from .dispatch.http2 import HTTP2Connection +from .dispatch.http11 import HTTP11Connection from .exceptions import ( ConnectTimeout, PoolTimeout, @@ -12,9 +14,9 @@ from .exceptions import ( StreamConsumed, Timeout, ) -from .http2 import HTTP2Connection -from .http11 import HTTP11Connection +from .interfaces import Adapter from .models import URL, Headers, Origin, Request, Response +from .status_codes import codes from .streams import BaseReader, BaseWriter, Protocol, Reader, Writer, connect from .sync import SyncClient, SyncConnectionPool diff --git a/httpcore/adapters/__init__.py b/httpcore/adapters/__init__.py new file mode 100644 index 00000000..8d4629f7 --- /dev/null +++ b/httpcore/adapters/__init__.py @@ -0,0 +1,4 @@ +""" +Adapter classes layer additional behavior over the raw dispatching of the +HTTP request/response. +""" diff --git a/httpcore/auth.py b/httpcore/adapters/authentication.py similarity index 78% rename from httpcore/auth.py rename to httpcore/adapters/authentication.py index 38c434a2..cb5ae99a 100644 --- a/httpcore/auth.py +++ b/httpcore/adapters/authentication.py @@ -1,10 +1,10 @@ import typing -from .adapters import Adapter -from .models import Request, Response +from ..interfaces import Adapter +from ..models import Request, Response -class AuthAdapter(Adapter): +class AuthenticationAdapter(Adapter): def __init__(self, dispatch: Adapter): self.dispatch = dispatch diff --git a/httpcore/cookies.py b/httpcore/adapters/cookies.py similarity index 85% rename from httpcore/cookies.py rename to httpcore/adapters/cookies.py index ed9e97c4..11051521 100644 --- a/httpcore/cookies.py +++ b/httpcore/adapters/cookies.py @@ -1,7 +1,7 @@ import typing -from .adapters import Adapter -from .models import Request, Response +from ..interfaces import Adapter +from ..models import Request, Response class CookieAdapter(Adapter): diff --git a/httpcore/environment.py b/httpcore/adapters/environment.py similarity index 90% rename from httpcore/environment.py rename to httpcore/adapters/environment.py index 5065eed8..9840a926 100644 --- a/httpcore/environment.py +++ b/httpcore/adapters/environment.py @@ -1,7 +1,7 @@ import typing -from .adapters import Adapter -from .models import Request, Response +from ..interfaces import Adapter +from ..models import Request, Response class EnvironmentAdapter(Adapter): diff --git a/httpcore/redirects.py b/httpcore/adapters/redirects.py similarity index 77% rename from httpcore/redirects.py rename to httpcore/adapters/redirects.py index 557b7468..0cf3f9fb 100644 --- a/httpcore/redirects.py +++ b/httpcore/adapters/redirects.py @@ -1,15 +1,16 @@ import typing from urllib.parse import urljoin, urlparse -from .adapters import Adapter -from .exceptions import TooManyRedirects -from .models import URL, Request, Response -from .status_codes import codes -from .utils import requote_uri +from ..config import DEFAULT_MAX_REDIRECTS +from ..exceptions import TooManyRedirects +from ..interfaces import Adapter +from ..models import URL, Request, Response +from ..status_codes import codes +from ..utils import requote_uri class RedirectAdapter(Adapter): - def __init__(self, dispatch: Adapter, max_redirects: int): + def __init__(self, dispatch: Adapter, max_redirects: int = DEFAULT_MAX_REDIRECTS): self.dispatch = dispatch self.max_redirects = max_redirects @@ -37,7 +38,7 @@ class RedirectAdapter(Adapter): def build_redirect_request(self, request: Request, response: Response) -> Request: method = self.redirect_method(request, response) url = self.redirect_url(request, response) - raise NotImplementedError() + return Request(method=method, url=url) def redirect_method(self, request: Request, response: Response) -> str: """ @@ -47,17 +48,17 @@ class RedirectAdapter(Adapter): method = request.method # https://tools.ietf.org/html/rfc7231#section-6.4.4 - if response.status_code == codes["see_other"] and method != "HEAD": + if response.status_code == codes.see_other and method != "HEAD": method = "GET" # Do what the browsers do, despite standards... - # First, turn 302s into GETs. - if response.status_code == codes["found"] and method != "HEAD": + # Turn 302s into GETs. + if response.status_code == codes.found and method != "HEAD": method = "GET" - # Second, if a POST is responded to with a 301, turn it into a GET. - # This bizarre behaviour is explained in Issue 1704. - if response.status_code == codes["moved"] and method == "POST": + # If a POST is responded to with a 301, turn it into a GET. + # This bizarre behaviour is explained in 'requests' issue 1704. + if response.status_code == codes.moved_permanently and method == "POST": method = "GET" return method diff --git a/httpcore/client.py b/httpcore/client.py index 2d98ba30..139a5686 100644 --- a/httpcore/client.py +++ b/httpcore/client.py @@ -1,7 +1,10 @@ import typing from types import TracebackType -from .auth import AuthAdapter +from .adapters.authentication import AuthenticationAdapter +from .adapters.cookies import CookieAdapter +from .adapters.environment import EnvironmentAdapter +from .adapters.redirects import RedirectAdapter from .config import ( DEFAULT_MAX_REDIRECTS, DEFAULT_POOL_LIMITS, @@ -11,11 +14,8 @@ from .config import ( SSLConfig, TimeoutConfig, ) -from .connection_pool import ConnectionPool -from .cookies import CookieAdapter -from .environment import EnvironmentAdapter +from .dispatch.connection_pool import ConnectionPool from .models import URL, Request, Response -from .redirects import RedirectAdapter class Client: @@ -28,7 +28,7 @@ class Client: ): connection_pool = ConnectionPool(ssl=ssl, timeout=timeout, limits=limits) cookie_adapter = CookieAdapter(dispatch=connection_pool) - auth_adapter = AuthAdapter(dispatch=cookie_adapter) + auth_adapter = AuthenticationAdapter(dispatch=cookie_adapter) redirect_adapter = RedirectAdapter( dispatch=auth_adapter, max_redirects=max_redirects ) diff --git a/httpcore/config.py b/httpcore/config.py index 5ce24707..ef24a8b1 100644 --- a/httpcore/config.py +++ b/httpcore/config.py @@ -175,4 +175,4 @@ DEFAULT_SSL_CONFIG = SSLConfig(cert=None, verify=True) DEFAULT_TIMEOUT_CONFIG = TimeoutConfig(timeout=5.0) DEFAULT_POOL_LIMITS = PoolLimits(soft_limit=10, hard_limit=100, pool_timeout=5.0) DEFAULT_CA_BUNDLE_PATH = certifi.where() -DEFAULT_MAX_REDIRECTS = 30 +DEFAULT_MAX_REDIRECTS = 20 diff --git a/httpcore/dispatch/__init__.py b/httpcore/dispatch/__init__.py new file mode 100644 index 00000000..4057d6ea --- /dev/null +++ b/httpcore/dispatch/__init__.py @@ -0,0 +1,4 @@ +""" +Dispatch classes handle the raw network connections and the implementation +details of making the HTTP request and receiving the response. +""" diff --git a/httpcore/connection.py b/httpcore/dispatch/connection.py similarity index 91% rename from httpcore/connection.py rename to httpcore/dispatch/connection.py index 1662018a..45134456 100644 --- a/httpcore/connection.py +++ b/httpcore/dispatch/connection.py @@ -4,13 +4,18 @@ import typing import h2.connection import h11 -from .adapters import Adapter -from .config import DEFAULT_SSL_CONFIG, DEFAULT_TIMEOUT_CONFIG, SSLConfig, TimeoutConfig -from .exceptions import ConnectTimeout +from ..config import ( + DEFAULT_SSL_CONFIG, + DEFAULT_TIMEOUT_CONFIG, + SSLConfig, + TimeoutConfig, +) +from ..exceptions import ConnectTimeout +from ..interfaces import Adapter +from ..models import Origin, Request, Response +from ..streams import Protocol, connect from .http2 import HTTP2Connection from .http11 import HTTP11Connection -from .models import Origin, Request, Response -from .streams import Protocol, connect # Callback signature: async def callback(conn: HTTPConnection) -> None ReleaseCallback = typing.Callable[["HTTPConnection"], typing.Awaitable[None]] diff --git a/httpcore/connection_pool.py b/httpcore/dispatch/connection_pool.py similarity index 96% rename from httpcore/connection_pool.py rename to httpcore/dispatch/connection_pool.py index ab394a19..a536d5f3 100644 --- a/httpcore/connection_pool.py +++ b/httpcore/dispatch/connection_pool.py @@ -1,8 +1,7 @@ import collections.abc import typing -from .adapters import Adapter -from .config import ( +from ..config import ( DEFAULT_CA_BUNDLE_PATH, DEFAULT_POOL_LIMITS, DEFAULT_SSL_CONFIG, @@ -11,10 +10,11 @@ from .config import ( SSLConfig, TimeoutConfig, ) +from ..exceptions import PoolTimeout +from ..interfaces import Adapter +from ..models import Origin, Request, Response +from ..streams import PoolSemaphore from .connection import HTTPConnection -from .exceptions import PoolTimeout -from .models import Origin, Request, Response -from .streams import PoolSemaphore CONNECTIONS_DICT = typing.Dict[Origin, typing.List[HTTPConnection]] diff --git a/httpcore/http11.py b/httpcore/dispatch/http11.py similarity index 93% rename from httpcore/http11.py rename to httpcore/dispatch/http11.py index 97e8b6a8..e4a5c83a 100644 --- a/httpcore/http11.py +++ b/httpcore/dispatch/http11.py @@ -2,11 +2,16 @@ import typing import h11 -from .adapters import Adapter -from .config import DEFAULT_SSL_CONFIG, DEFAULT_TIMEOUT_CONFIG, SSLConfig, TimeoutConfig -from .exceptions import ConnectTimeout, ReadTimeout -from .models import Request, Response -from .streams import BaseReader, BaseWriter +from ..config import ( + DEFAULT_SSL_CONFIG, + DEFAULT_TIMEOUT_CONFIG, + SSLConfig, + TimeoutConfig, +) +from ..exceptions import ConnectTimeout, ReadTimeout +from ..interfaces import Adapter +from ..models import Request, Response +from ..streams import BaseReader, BaseWriter H11Event = typing.Union[ h11.Request, diff --git a/httpcore/http2.py b/httpcore/dispatch/http2.py similarity index 94% rename from httpcore/http2.py rename to httpcore/dispatch/http2.py index 40448176..7bd124b7 100644 --- a/httpcore/http2.py +++ b/httpcore/dispatch/http2.py @@ -4,11 +4,16 @@ import typing import h2.connection import h2.events -from .adapters import Adapter -from .config import DEFAULT_SSL_CONFIG, DEFAULT_TIMEOUT_CONFIG, SSLConfig, TimeoutConfig -from .exceptions import ConnectTimeout, ReadTimeout -from .models import Request, Response -from .streams import BaseReader, BaseWriter +from ..config import ( + DEFAULT_SSL_CONFIG, + DEFAULT_TIMEOUT_CONFIG, + SSLConfig, + TimeoutConfig, +) +from ..exceptions import ConnectTimeout, ReadTimeout +from ..interfaces import Adapter +from ..models import Request, Response +from ..streams import BaseReader, BaseWriter OptionalTimeout = typing.Optional[TimeoutConfig] diff --git a/httpcore/adapters.py b/httpcore/interfaces.py similarity index 60% rename from httpcore/adapters.py rename to httpcore/interfaces.py index 72d5880f..d3cdca47 100644 --- a/httpcore/adapters.py +++ b/httpcore/interfaces.py @@ -1,8 +1,11 @@ import typing from types import TracebackType +from .config import TimeoutConfig from .models import URL, Request, Response +OptionalTimeout = typing.Optional[TimeoutConfig] + class Adapter: async def request( @@ -38,3 +41,27 @@ class Adapter: traceback: TracebackType = None, ) -> None: await self.close() + + +class BaseReader: + async def read(self, n: int, timeout: OptionalTimeout = None) -> bytes: + raise NotImplementedError() # pragma: no cover + + +class BaseWriter: + def write_no_block(self, data: bytes) -> None: + raise NotImplementedError() # pragma: no cover + + async def write(self, data: bytes, timeout: OptionalTimeout = None) -> None: + raise NotImplementedError() # pragma: no cover + + async def close(self) -> None: + raise NotImplementedError() # pragma: no cover + + +class BasePoolSemaphore: + async def acquire(self) -> None: + raise NotImplementedError() # pragma: no cover + + def release(self) -> None: + raise NotImplementedError() # pragma: no cover diff --git a/httpcore/models.py b/httpcore/models.py index 791bbc90..b08da934 100644 --- a/httpcore/models.py +++ b/httpcore/models.py @@ -70,6 +70,9 @@ class URL: def origin(self) -> "Origin": return Origin(self) + def __str__(self) -> str: + return self.components.geturl() + class Origin: def __init__(self, url: typing.Union[str, URL]) -> None: diff --git a/httpcore/status_codes.py b/httpcore/status_codes.py index 9265a71b..19b849f0 100644 --- a/httpcore/status_codes.py +++ b/httpcore/status_codes.py @@ -1,123 +1,58 @@ -""" -The ``codes`` object defines a mapping from common names for HTTP statuses -to their numerical codes, accessible either as attributes or as dictionary -items. ->>> requests.codes['temporary_redirect'] -307 ->>> requests.codes.teapot -418 -Some codes have multiple names, and both upper- and lower-case versions of -the names are allowed. For example, ``codes.ok``, ``codes.OK``, and -``codes.okay`` all correspond to the HTTP status code 200. -""" - -import typing - -from .structures import LookupDict - -_codes = { - # Informational. - 100: ("continue",), - 101: ("switching_protocols",), - 102: ("processing",), - 103: ("checkpoint",), - 122: ("uri_too_long", "request_uri_too_long"), - 200: ("ok", "okay", "all_ok", "all_okay", "all_good", "\\o/", "✓"), - 201: ("created",), - 202: ("accepted",), - 203: ("non_authoritative_info", "non_authoritative_information"), - 204: ("no_content",), - 205: ("reset_content", "reset"), - 206: ("partial_content", "partial"), - 207: ("multi_status", "multiple_status", "multi_stati", "multiple_stati"), - 208: ("already_reported",), - 226: ("im_used",), - # Redirection. - 300: ("multiple_choices",), - 301: ("moved_permanently", "moved", "\\o-"), - 302: ("found",), - 303: ("see_other", "other"), - 304: ("not_modified",), - 305: ("use_proxy",), - 306: ("switch_proxy",), - 307: ("temporary_redirect", "temporary_moved", "temporary"), - 308: ( - "permanent_redirect", - "resume_incomplete", - "resume", - ), # These 2 to be removed in 3.0 - # Client Error. - 400: ("bad_request", "bad"), - 401: ("unauthorized",), - 402: ("payment_required", "payment"), - 403: ("forbidden",), - 404: ("not_found", "-o-"), - 405: ("method_not_allowed", "not_allowed"), - 406: ("not_acceptable",), - 407: ("proxy_authentication_required", "proxy_auth", "proxy_authentication"), - 408: ("request_timeout", "timeout"), - 409: ("conflict",), - 410: ("gone",), - 411: ("length_required",), - 412: ("precondition_failed", "precondition"), - 413: ("request_entity_too_large",), - 414: ("request_uri_too_large",), - 415: ("unsupported_media_type", "unsupported_media", "media_type"), - 416: ( - "requested_range_not_satisfiable", - "requested_range", - "range_not_satisfiable", - ), - 417: ("expectation_failed",), - 418: ("im_a_teapot", "teapot", "i_am_a_teapot"), - 421: ("misdirected_request",), - 422: ("unprocessable_entity", "unprocessable"), - 423: ("locked",), - 424: ("failed_dependency", "dependency"), - 425: ("unordered_collection", "unordered"), - 426: ("upgrade_required", "upgrade"), - 428: ("precondition_required", "precondition"), - 429: ("too_many_requests", "too_many"), - 431: ("header_fields_too_large", "fields_too_large"), - 444: ("no_response", "none"), - 449: ("retry_with", "retry"), - 450: ("blocked_by_windows_parental_controls", "parental_controls"), - 451: ("unavailable_for_legal_reasons", "legal_reasons"), - 499: ("client_closed_request",), - # Server Error. - 500: ("internal_server_error", "server_error", "/o\\", "✗"), - 501: ("not_implemented",), - 502: ("bad_gateway",), - 503: ("service_unavailable", "unavailable"), - 504: ("gateway_timeout",), - 505: ("http_version_not_supported", "http_version"), - 506: ("variant_also_negotiates",), - 507: ("insufficient_storage",), - 509: ("bandwidth_limit_exceeded", "bandwidth"), - 510: ("not_extended",), - 511: ("network_authentication_required", "network_auth", "network_authentication"), -} # type: typing.Dict[int, typing.Sequence[str]] - -codes = LookupDict(name="status_codes") - - -def _init() -> None: - for code, titles in _codes.items(): - for title in titles: - setattr(codes, title, code) - if not title.startswith(("\\", "/")): - setattr(codes, title.upper(), code) - - def doc(code: int) -> str: - names = ", ".join("``%s``" % n for n in _codes[code]) - return "* %d: %s" % (code, names) - - global __doc__ - __doc__ = ( - __doc__ + "\n" + "\n".join(doc(code) for code in sorted(_codes)) - if __doc__ is not None - else None - ) - - -_init() +import enum + +codes = enum.IntEnum( + "StatusCode", + [ + ("continue", 100), + ("switching_protocols", 101), + ("ok", 200), + ("created", 201), + ("accepted", 202), + ("non_authoritative_information", 203), + ("no_content", 204), + ("reset_content", 205), + ("partial_content", 206), + ("multi_status", 207), + ("multiple_choices", 300), + ("moved_permanently", 301), + ("found", 302), + ("see_other", 303), + ("not_modified", 304), + ("use_proxy", 305), + ("reserved", 306), + ("temporary_redirect", 307), + ("bad_request", 400), + ("unauthorized", 401), + ("payment_required", 402), + ("forbidden", 403), + ("not_found", 404), + ("method_not_allowed", 405), + ("not_acceptable", 406), + ("proxy_authentication_required", 407), + ("request_timeout", 408), + ("conflict", 409), + ("gone", 410), + ("length_required", 411), + ("precondition_failed", 412), + ("request_entity_too_large", 413), + ("request_uri_too_long", 414), + ("unsupported_media_type", 415), + ("requested_range_not_satisfiable", 416), + ("expectation_failed", 417), + ("unprocessable_entity", 422), + ("locked", 423), + ("failed_dependency", 424), + ("precondition_required", 428), + ("too_many_requests", 429), + ("request_header_fields_too_large", 431), + ("unavailable_for_legal_reasons", 451), + ("internal_server_error", 500), + ("not_implemented", 501), + ("bad_gateway", 502), + ("service_unavailable", 503), + ("gateway_timeout", 504), + ("http_version_not_supported", 505), + ("insufficient_storage", 507), + ("network_authentication_required", 511), + ], +) diff --git a/httpcore/streams.py b/httpcore/streams.py index 03bba17c..a8590db4 100644 --- a/httpcore/streams.py +++ b/httpcore/streams.py @@ -15,6 +15,7 @@ import typing from .config import DEFAULT_TIMEOUT_CONFIG, PoolLimits, TimeoutConfig from .exceptions import ConnectTimeout, PoolTimeout, ReadTimeout, WriteTimeout +from .interfaces import BasePoolSemaphore, BaseReader, BaseWriter OptionalTimeout = typing.Optional[TimeoutConfig] @@ -24,30 +25,6 @@ class Protocol(enum.Enum): HTTP_2 = 2 -class BaseReader: - async def read(self, n: int, timeout: OptionalTimeout = None) -> bytes: - raise NotImplementedError() # pragma: no cover - - -class BaseWriter: - def write_no_block(self, data: bytes) -> None: - raise NotImplementedError() # pragma: no cover - - async def write(self, data: bytes, timeout: OptionalTimeout = None) -> None: - raise NotImplementedError() # pragma: no cover - - async def close(self) -> None: - raise NotImplementedError() # pragma: no cover - - -class BasePoolSemaphore: - async def acquire(self) -> None: - raise NotImplementedError() # pragma: no cover - - def release(self) -> None: - raise NotImplementedError() # pragma: no cover - - class Reader(BaseReader): def __init__( self, stream_reader: asyncio.StreamReader, timeout: TimeoutConfig diff --git a/httpcore/structures.py b/httpcore/structures.py deleted file mode 100644 index 127af5c6..00000000 --- a/httpcore/structures.py +++ /dev/null @@ -1,20 +0,0 @@ -import typing - - -class LookupDict(dict): - """Dictionary lookup object.""" - - def __init__(self, name: str = None) -> None: - self.name = name - super(LookupDict, self).__init__() - - def __repr__(self) -> str: - return "" % (self.name) - - def __getitem__(self, key: typing.Any) -> typing.Any: - # We allow fall-through here, so values default to None - - return self.__dict__.get(key, None) - - def get(self, key: typing.Any, default: typing.Any = None) -> typing.Any: - return self.__dict__.get(key, default) diff --git a/httpcore/sync.py b/httpcore/sync.py index 89b58628..2d58f9a1 100644 --- a/httpcore/sync.py +++ b/httpcore/sync.py @@ -2,9 +2,9 @@ import asyncio import typing from types import TracebackType -from .adapters import Adapter from .config import SSLConfig, TimeoutConfig -from .connection_pool import ConnectionPool +from .dispatch.connection_pool import ConnectionPool +from .interfaces import Adapter from .models import URL, Headers, Response diff --git a/httpcore/utils.py b/httpcore/utils.py index dc7f1d28..cd11858a 100644 --- a/httpcore/utils.py +++ b/httpcore/utils.py @@ -9,9 +9,9 @@ UNRESERVED_SET = frozenset( def unquote_unreserved(uri: str) -> str: - """Un-escape any percent-escape sequences in a URI that are unreserved + """ + Un-escape any percent-escape sequences in a URI that are unreserved characters. This leaves all reserved, illegal and non-ASCII bytes encoded. - :rtype: str """ parts = uri.split("%") for i in range(1, len(parts)): diff --git a/tests/adapters/test_redirects.py b/tests/adapters/test_redirects.py new file mode 100644 index 00000000..f4a98105 --- /dev/null +++ b/tests/adapters/test_redirects.py @@ -0,0 +1,31 @@ +import pytest + +from httpcore import Adapter, RedirectAdapter, Request, Response, codes + + +class MockDispatch(Adapter): + def prepare_request(self, request: Request) -> None: + pass + + async def send(self, request: Request, **options) -> Response: + if request.url.path == "/redirect_303": + return Response( + codes.see_other, headers=[(b"location", b"https://example.org/")] + ) + elif request.url.path == "/relative_redirect": + return Response(codes.see_other, headers=[(b"location", b"/")]) + return Response(codes.ok, body=b"Hello, world!") + + +@pytest.mark.asyncio +async def test_redirect_303(): + client = RedirectAdapter(MockDispatch()) + response = await client.request("GET", "https://example.org/redirect_303") + assert response.status_code == codes.ok + + +@pytest.mark.asyncio +async def test_relative_redirect(): + client = RedirectAdapter(MockDispatch()) + response = await client.request("GET", "https://example.org/relative_redirect") + assert response.status_code == codes.ok