From: Tom Christie Date: Fri, 13 Aug 2021 10:52:45 +0000 (+0100) Subject: Use either brotli or brotlicffi. (#1618) X-Git-Tag: 0.19.0~5 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=d5143120d10f19cf2876179c9d80eaeac631bb52;p=thirdparty%2Fhttpx.git Use either brotli or brotlicffi. (#1618) * Use either brotli (recommended for CPython) or brotlicffi (Recommended for PyPy and others) * Add comments in places where we switch behaviour depending on brotli/brotlicffi Co-authored-by: Florimond Manca --- diff --git a/README.md b/README.md index 54c496fa..268a8512 100644 --- a/README.md +++ b/README.md @@ -124,7 +124,7 @@ The HTTPX project relies on these excellent libraries: * `idna` - Internationalized domain name support. * `sniffio` - Async library autodetection. * `async_generator` - Backport support for `contextlib.asynccontextmanager`. *(Only required for Python 3.6)* -* `brotlicffi` - Decoding for "brotli" compressed responses. *(Optional)* +* `brotli` or `brotlicffi` - Decoding for "brotli" compressed responses. *(Optional)* A huge amount of credit is due to `requests` for the API layout that much of this work follows, as well as to `urllib3` for plenty of design diff --git a/docs/index.md b/docs/index.md index 3a00ca8c..b6476859 100644 --- a/docs/index.md +++ b/docs/index.md @@ -116,7 +116,7 @@ The HTTPX project relies on these excellent libraries: * `idna` - Internationalized domain name support. * `sniffio` - Async library autodetection. * `async_generator` - Backport support for `contextlib.asynccontextmanager`. *(Only required for Python 3.6)* -* `brotlicffi` - Decoding for "brotli" compressed responses. *(Optional)* +* `brotli` or `brotlicffi` - Decoding for "brotli" compressed responses. *(Optional)* A huge amount of credit is due to `requests` for the API layout that much of this work follows, as well as to `urllib3` for plenty of design diff --git a/httpx/_compat.py b/httpx/_compat.py index a499322f..15e915a9 100644 --- a/httpx/_compat.py +++ b/httpx/_compat.py @@ -12,6 +12,17 @@ try: except ImportError: from async_generator import asynccontextmanager # type: ignore # noqa +# Brotli support is optional +# The C bindings in `brotli` are recommended for CPython. +# The CFFI bindings in `brotlicffi` are recommended for PyPy and everything else. +try: + import brotlicffi as brotli +except ImportError: # pragma: nocover + try: + import brotli + except ImportError: + brotli = None + if sys.version_info >= (3, 10) or ( sys.version_info >= (3, 7) and ssl.OPENSSL_VERSION_INFO >= (1, 1, 0, 7) ): diff --git a/httpx/_decoders.py b/httpx/_decoders.py index 50e6798e..5081c86f 100644 --- a/httpx/_decoders.py +++ b/httpx/_decoders.py @@ -8,13 +8,9 @@ import io import typing import zlib +from ._compat import brotli from ._exceptions import DecodingError -try: - import brotlicffi -except ImportError: # pragma: nocover - brotlicffi = None - class ContentDecoder: def decode(self, data: bytes) -> bytes: @@ -99,18 +95,20 @@ class BrotliDecoder(ContentDecoder): """ def __init__(self) -> None: - if brotlicffi is None: # pragma: nocover + if brotli is None: # pragma: nocover raise ImportError( - "Using 'BrotliDecoder', but the 'brotlicffi' library " - "is not installed." + "Using 'BrotliDecoder', but neither of the 'brotlicffi' or 'brotli' " + "packages have been installed. " "Make sure to install httpx using `pip install httpx[brotli]`." ) from None - self.decompressor = brotlicffi.Decompressor() + self.decompressor = brotli.Decompressor() self.seen_data = False if hasattr(self.decompressor, "decompress"): - self._decompress = self.decompressor.decompress + # The 'brotlicffi' package. + self._decompress = self.decompressor.decompress # pragma: nocover else: + # The 'brotli' package. self._decompress = self.decompressor.process # pragma: nocover def decode(self, data: bytes) -> bytes: @@ -118,8 +116,8 @@ class BrotliDecoder(ContentDecoder): return b"" self.seen_data = True try: - return self.decompressor.decompress(data) - except brotlicffi.Error as exc: + return self._decompress(data) + except brotli.error as exc: raise DecodingError(str(exc)) from exc def flush(self) -> bytes: @@ -127,9 +125,14 @@ class BrotliDecoder(ContentDecoder): return b"" try: if hasattr(self.decompressor, "finish"): - self.decompressor.finish() + # Only available in the 'brotlicffi' package. + + # As the decompressor decompresses eagerly, this + # will never actually emit any data. However, it will potentially throw + # errors if a truncated or damaged data stream has been used. + self.decompressor.finish() # pragma: nocover return b"" - except brotlicffi.Error as exc: # pragma: nocover + except brotli.error as exc: # pragma: nocover raise DecodingError(str(exc)) from exc @@ -326,5 +329,5 @@ SUPPORTED_DECODERS = { } -if brotlicffi is None: +if brotli is None: SUPPORTED_DECODERS.pop("br") # pragma: nocover diff --git a/setup.py b/setup.py index aeee7c6c..243bbd83 100644 --- a/setup.py +++ b/setup.py @@ -65,7 +65,10 @@ setup( ], extras_require={ "http2": "h2>=3,<5", - "brotli": "brotlicffi==1.*", + "brotli": [ + "brotli; platform_python_implementation == 'CPython'", + "brotlicffi; platform_python_implementation != 'CPython'" + ], }, classifiers=[ "Development Status :: 4 - Beta", diff --git a/tests/models/test_responses.py b/tests/models/test_responses.py index b9cc0773..adcf2f65 100644 --- a/tests/models/test_responses.py +++ b/tests/models/test_responses.py @@ -1,10 +1,10 @@ import json import pickle -import brotlicffi import pytest import httpx +from httpx._compat import brotli class StreamingBody: @@ -798,7 +798,7 @@ def test_link_headers(headers, expected): def test_decode_error_with_request(header_value): headers = [(b"Content-Encoding", header_value)] body = b"test 123" - compressed_body = brotlicffi.compress(body)[3:] + compressed_body = brotli.compress(body)[3:] with pytest.raises(httpx.DecodingError): httpx.Response( 200, @@ -819,7 +819,7 @@ def test_decode_error_with_request(header_value): def test_value_error_without_request(header_value): headers = [(b"Content-Encoding", header_value)] body = b"test 123" - compressed_body = brotlicffi.compress(body)[3:] + compressed_body = brotli.compress(body)[3:] with pytest.raises(httpx.DecodingError): httpx.Response(200, headers=headers, content=compressed_body) diff --git a/tests/test_decoders.py b/tests/test_decoders.py index 4f182efd..f31abf09 100644 --- a/tests/test_decoders.py +++ b/tests/test_decoders.py @@ -1,9 +1,9 @@ import zlib -import brotlicffi import pytest import httpx +from httpx._compat import brotli from httpx._decoders import ( BrotliDecoder, ByteChunker, @@ -69,7 +69,7 @@ def test_gzip(): def test_brotli(): body = b"test 123" - compressed_body = brotlicffi.compress(body) + compressed_body = brotli.compress(body) headers = [(b"Content-Encoding", b"br")] response = httpx.Response( @@ -102,7 +102,7 @@ def test_multi(): def test_multi_with_identity(): body = b"test 123" - compressed_body = brotlicffi.compress(body) + compressed_body = brotli.compress(body) headers = [(b"Content-Encoding", b"br, identity")] response = httpx.Response( @@ -165,7 +165,7 @@ def test_decoders_empty_cases(decoder): def test_decoding_errors(header_value): headers = [(b"Content-Encoding", header_value)] body = b"test 123" - compressed_body = brotlicffi.compress(body)[3:] + compressed_body = brotli.compress(body)[3:] with pytest.raises(httpx.DecodingError): request = httpx.Request("GET", "https://example.org") httpx.Response(200, headers=headers, content=compressed_body, request=request)