Use either brotli or brotlicffi. (#1618)

* Use either brotli (recommended for CPython) or brotlicffi (Recommended for PyPy and others)

* Add comments in places where we switch behaviour depending on brotli/brotlicffi

Co-authored-by: Florimond Manca <[email protected]>

Author: tomchristie

README.md

@@ -124,7 +124,7 @@ The HTTPX project relies on these excellent libraries:
   * `idna` - Internationalized domain name support.
 * `sniffio` - Async library autodetection.
 * `async_generator` - Backport support for `contextlib.asynccontextmanager`. *(Only required for Python 3.6)*
-* `brotlicffi` - Decoding for "brotli" compressed responses. *(Optional)*
+* `brotli` or `brotlicffi` - Decoding for "brotli" compressed responses. *(Optional)*
 
 A huge amount of credit is due to `requests` for the API layout that
 much of this work follows, as well as to `urllib3` for plenty of design

docs/index.md

@@ -116,7 +116,7 @@ The HTTPX project relies on these excellent libraries:
   * `idna` - Internationalized domain name support.
 * `sniffio` - Async library autodetection.
 * `async_generator` - Backport support for `contextlib.asynccontextmanager`. *(Only required for Python 3.6)*
-* `brotlicffi` - Decoding for "brotli" compressed responses. *(Optional)*
+* `brotli` or `brotlicffi` - Decoding for "brotli" compressed responses. *(Optional)*
 
 A huge amount of credit is due to `requests` for the API layout that
 much of this work follows, as well as to `urllib3` for plenty of design

httpx/_compat.py

@@ -12,6 +12,17 @@
 except ImportError:
     from async_generator import asynccontextmanager  # type: ignore # noqa
 
+# Brotli support is optional
+# The C bindings in `brotli` are recommended for CPython.
+# The CFFI bindings in `brotlicffi` are recommended for PyPy and everything else.
+try:
+    import brotlicffi as brotli
+except ImportError:  # pragma: nocover
+    try:
+        import brotli
+    except ImportError:
+        brotli = None
+
 if sys.version_info >= (3, 10) or (
     sys.version_info >= (3, 7) and ssl.OPENSSL_VERSION_INFO >= (1, 1, 0, 7)
 ):

httpx/_decoders.py

@@ -8,13 +8,9 @@
 import typing
 import zlib
 
+from ._compat import brotli
 from ._exceptions import DecodingError
 
-try:
-    import brotlicffi
-except ImportError:  # pragma: nocover
-    brotlicffi = None
-
 
 class ContentDecoder:
     def decode(self, data: bytes) -> bytes:
@@ -99,37 +95,44 @@ class BrotliDecoder(ContentDecoder):
     """
 
     def __init__(self) -> None:
-        if brotlicffi is None:  # pragma: nocover
+        if brotli is None:  # pragma: nocover
             raise ImportError(
-                "Using 'BrotliDecoder', but the 'brotlicffi' library "
-                "is not installed."
+                "Using 'BrotliDecoder', but neither of the 'brotlicffi' or 'brotli' "
+                "packages have been installed. "
                 "Make sure to install httpx using `pip install httpx[brotli]`."
             ) from None
 
-        self.decompressor = brotlicffi.Decompressor()
+        self.decompressor = brotli.Decompressor()
         self.seen_data = False
         if hasattr(self.decompressor, "decompress"):
-            self._decompress = self.decompressor.decompress
+            # The 'brotlicffi' package.
+            self._decompress = self.decompressor.decompress  # pragma: nocover
         else:
+            # The 'brotli' package.
             self._decompress = self.decompressor.process  # pragma: nocover
 
     def decode(self, data: bytes) -> bytes:
         if not data:
             return b""
         self.seen_data = True
         try:
-            return self.decompressor.decompress(data)
-        except brotlicffi.Error as exc:
+            return self._decompress(data)
+        except brotli.error as exc:
             raise DecodingError(str(exc)) from exc
 
     def flush(self) -> bytes:
         if not self.seen_data:
             return b""
         try:
             if hasattr(self.decompressor, "finish"):
-                self.decompressor.finish()
+                # Only available in the 'brotlicffi' package.
+
+                # As the decompressor decompresses eagerly, this
+                # will never actually emit any data. However, it will potentially throw
+                # errors if a truncated or damaged data stream has been used.
+                self.decompressor.finish()  # pragma: nocover
             return b""
-        except brotlicffi.Error as exc:  # pragma: nocover
+        except brotli.error as exc:  # pragma: nocover
             raise DecodingError(str(exc)) from exc
 
 
@@ -326,5 +329,5 @@ def flush(self) -> typing.List[str]:
 }
 
 
-if brotlicffi is None:
+if brotli is None:
     SUPPORTED_DECODERS.pop("br")  # pragma: nocover

setup.py

@@ -65,7 +65,10 @@ def get_packages(package):
     ],
     extras_require={
         "http2": "h2>=3,<5",
-        "brotli": "brotlicffi==1.*",
+        "brotli": [
+            "brotli; platform_python_implementation == 'CPython'",
+            "brotlicffi; platform_python_implementation != 'CPython'"
+        ],
     },
     classifiers=[
         "Development Status :: 4 - Beta",

tests/models/test_responses.py

@@ -1,10 +1,10 @@
 import json
 import pickle
 
-import brotlicffi
 import pytest
 
 import httpx
+from httpx._compat import brotli
 
 
 class StreamingBody:
@@ -798,7 +798,7 @@ def test_link_headers(headers, expected):
 def test_decode_error_with_request(header_value):
     headers = [(b"Content-Encoding", header_value)]
     body = b"test 123"
-    compressed_body = brotlicffi.compress(body)[3:]
+    compressed_body = brotli.compress(body)[3:]
     with pytest.raises(httpx.DecodingError):
         httpx.Response(
             200,
@@ -819,7 +819,7 @@ def test_decode_error_with_request(header_value):
 def test_value_error_without_request(header_value):
     headers = [(b"Content-Encoding", header_value)]
     body = b"test 123"
-    compressed_body = brotlicffi.compress(body)[3:]
+    compressed_body = brotli.compress(body)[3:]
     with pytest.raises(httpx.DecodingError):
         httpx.Response(200, headers=headers, content=compressed_body)
 

tests/test_decoders.py

@@ -1,9 +1,9 @@
 import zlib
 
-import brotlicffi
 import pytest
 
 import httpx
+from httpx._compat import brotli
 from httpx._decoders import (
     BrotliDecoder,
     ByteChunker,
@@ -69,7 +69,7 @@ def test_gzip():
 
 def test_brotli():
     body = b"test 123"
-    compressed_body = brotlicffi.compress(body)
+    compressed_body = brotli.compress(body)
 
     headers = [(b"Content-Encoding", b"br")]
     response = httpx.Response(
@@ -102,7 +102,7 @@ def test_multi():
 
 def test_multi_with_identity():
     body = b"test 123"
-    compressed_body = brotlicffi.compress(body)
+    compressed_body = brotli.compress(body)
 
     headers = [(b"Content-Encoding", b"br, identity")]
     response = httpx.Response(
@@ -165,7 +165,7 @@ def test_decoders_empty_cases(decoder):
 def test_decoding_errors(header_value):
     headers = [(b"Content-Encoding", header_value)]
     body = b"test 123"
-    compressed_body = brotlicffi.compress(body)[3:]
+    compressed_body = brotli.compress(body)[3:]
     with pytest.raises(httpx.DecodingError):
         request = httpx.Request("GET", "https://example.org")
         httpx.Response(200, headers=headers, content=compressed_body, request=request)