Backport PR #54633 on branch 2.1.x (DEP: remove python-snappy and brotli as optional dependencies (no longer used)) (#54637)

Backport PR #54633: DEP: remove python-snappy and brotli as optional dependencies (no longer used)

Co-authored-by: Joris Van den Bossche <[email protected]>

Author: meeseeksmachine

ci/deps/actions-310.yaml

@@ -27,7 +27,6 @@ dependencies:
   - beautifulsoup4>=4.11.1
   - blosc>=1.21.0
   - bottleneck>=1.3.4
-  - brotlipy>=0.7.0
   - fastparquet>=0.8.1
   - fsspec>=2022.05.0
   - html5lib>=1.1
@@ -47,7 +46,6 @@ dependencies:
   - pymysql>=1.0.2
   - pyreadstat>=1.1.5
   - pytables>=3.7.0
-  - python-snappy>=0.6.1
   - pyxlsb>=1.0.9
   - s3fs>=2022.05.0
   - scipy>=1.8.1

ci/deps/actions-311-downstream_compat.yaml

@@ -28,7 +28,6 @@ dependencies:
   - beautifulsoup4>=4.11.1
   - blosc>=1.21.0
   - bottleneck>=1.3.4
-  - brotlipy>=0.7.0
   - fastparquet>=0.8.1
   - fsspec>=2022.05.0
   - html5lib>=1.1
@@ -48,7 +47,6 @@ dependencies:
   - pymysql>=1.0.2
   - pyreadstat>=1.1.5
   - pytables>=3.7.0
-  - python-snappy>=0.6.1
   - pyxlsb>=1.0.9
   - s3fs>=2022.05.0
   - scipy>=1.8.1

ci/deps/actions-311.yaml

@@ -27,7 +27,6 @@ dependencies:
   - beautifulsoup4>=4.11.1
   - blosc>=1.21.0
   - bottleneck>=1.3.4
-  - brotlipy>=0.7.0
   - fastparquet>=0.8.1
   - fsspec>=2022.05.0
   - html5lib>=1.1
@@ -47,7 +46,6 @@ dependencies:
   - pymysql>=1.0.2
   - pyreadstat>=1.1.5
   # - pytables>=3.7.0, 3.8.0 is first version that supports 3.11
-  - python-snappy>=0.6.1
   - pyxlsb>=1.0.9
   - s3fs>=2022.05.0
   - scipy>=1.8.1

ci/deps/actions-39-minimum_versions.yaml

@@ -29,7 +29,6 @@ dependencies:
   - beautifulsoup4=4.11.1
   - blosc=1.21.0
   - bottleneck=1.3.4
-  - brotlipy=0.7.0
   - fastparquet=0.8.1
   - fsspec=2022.05.0
   - html5lib=1.1
@@ -49,7 +48,6 @@ dependencies:
   - pymysql=1.0.2
   - pyreadstat=1.1.5
   - pytables=3.7.0
-  - python-snappy=0.6.1
   - pyxlsb=1.0.9
   - s3fs=2022.05.0
   - scipy=1.8.1

ci/deps/actions-39.yaml

@@ -27,7 +27,6 @@ dependencies:
   - beautifulsoup4>=4.11.1
   - blosc>=1.21.0
   - bottleneck>=1.3.4
-  - brotlipy>=0.7.0
   - fastparquet>=0.8.1
   - fsspec>=2022.05.0
   - html5lib>=1.1
@@ -47,7 +46,6 @@ dependencies:
   - pymysql>=1.0.2
   - pyreadstat>=1.1.5
   - pytables>=3.7.0
-  - python-snappy>=0.6.1
   - pyxlsb>=1.0.9
   - s3fs>=2022.05.0
   - scipy>=1.8.1

ci/deps/circle-310-arm64.yaml

@@ -27,7 +27,6 @@ dependencies:
   - beautifulsoup4>=4.11.1
   - blosc>=1.21.0
   - bottleneck>=1.3.4
-  - brotlipy>=0.7.0
   - fastparquet>=0.8.1
   - fsspec>=2022.05.0
   - html5lib>=1.1
@@ -48,7 +47,6 @@ dependencies:
   - pymysql>=1.0.2
   # - pyreadstat>=1.1.5 not available on ARM
   - pytables>=3.7.0
-  - python-snappy>=0.6.1
   - pyxlsb>=1.0.9
   - s3fs>=2022.05.0
   - scipy>=1.8.1

doc/source/getting_started/install.rst

@@ -412,8 +412,6 @@ Installable with ``pip install "pandas[compression]"``
 ========================= ================== =============== =============================================================
 Dependency                Minimum Version    pip extra       Notes
 ========================= ================== =============== =============================================================
-brotli                    0.7.0              compression     Brotli compression
-python-snappy             0.6.1              compression     Snappy compression
 Zstandard                 0.17.0             compression     Zstandard compression
 ========================= ================== =============== =============================================================
 

environment.yml

@@ -27,7 +27,6 @@ dependencies:
   # optional dependencies
   - beautifulsoup4>=4.11.1
   - blosc
-  - brotlipy>=0.7.0
   - bottleneck>=1.3.4
   - fastparquet>=0.8.1
   - fsspec>=2022.05.0
@@ -48,7 +47,6 @@ dependencies:
   - pymysql>=1.0.2
   - pyreadstat>=1.1.5
   - pytables>=3.7.0
-  - python-snappy>=0.6.1
   - pyxlsb>=1.0.9
   - s3fs>=2022.05.0
   - scipy>=1.8.1

pandas/compat/_optional.py

@@ -18,7 +18,6 @@
     "bs4": "4.11.1",
     "blosc": "1.21.0",
     "bottleneck": "1.3.4",
-    "brotli": "0.7.0",
     "dataframe-api-compat": "0.1.7",
     "fastparquet": "0.8.1",
     "fsspec": "2022.05.0",
@@ -41,7 +40,6 @@
     "pyxlsb": "1.0.9",
     "s3fs": "2022.05.0",
     "scipy": "1.8.1",
-    "snappy": "0.6.1",
     "sqlalchemy": "1.4.36",
     "tables": "3.7.0",
     "tabulate": "0.8.10",
@@ -60,12 +58,10 @@
 INSTALL_MAPPING = {
     "bs4": "beautifulsoup4",
     "bottleneck": "Bottleneck",
-    "brotli": "brotlipy",
     "jinja2": "Jinja2",
     "lxml.etree": "lxml",
     "odf": "odfpy",
     "pandas_gbq": "pandas-gbq",
-    "snappy": "python-snappy",
     "sqlalchemy": "SQLAlchemy",
     "tables": "pytables",
 }
@@ -75,13 +71,6 @@ def get_version(module: types.ModuleType) -> str:
     version = getattr(module, "__version__", None)
 
     if version is None:
-        if module.__name__ == "brotli":
-            # brotli doesn't contain attributes to confirm it's version
-            return ""
-        if module.__name__ == "snappy":
-            # snappy doesn't contain attributes to confirm it's version
-            # See https://github.com/andrix/python-snappy/pull/119
-            return ""
         raise ImportError(f"Can't determine version for {module.__name__}")
     if module.__name__ == "psycopg2":
         # psycopg2 appends " (dt dec pq3 ext lo64)" to it's version

pandas/core/frame.py

@@ -2909,10 +2909,7 @@ def to_parquet(
             'pyarrow' is unavailable.
         compression : str or None, default 'snappy'
             Name of the compression to use. Use ``None`` for no compression.
-            The supported compression methods actually depend on which engine
-            is used. For 'pyarrow', 'snappy', 'gzip', 'brotli', 'lz4', 'zstd'
-            are all supported. For 'fastparquet', only 'gzip' and 'snappy' are
-            supported.
+            Supported options: 'snappy', 'gzip', 'brotli', 'lz4', 'zstd'.
         index : bool, default None
             If ``True``, include the dataframe's index(es) in the file output.
             If ``False``, they will not be written to the file.

pandas/io/parquet.py

@@ -444,10 +444,7 @@ def to_parquet(
         if you wish to use its implementation.
     compression : {{'snappy', 'gzip', 'brotli', 'lz4', 'zstd', None}},
         default 'snappy'. Name of the compression to use. Use ``None``
-        for no compression. The supported compression methods actually
-        depend on which engine is used. For 'pyarrow', 'snappy', 'gzip',
-        'brotli', 'lz4', 'zstd' are all supported. For 'fastparquet',
-        only 'gzip' and 'snappy' are supported.
+        for no compression.
     index : bool, default None
         If ``True``, include the dataframe's index(es) in the file output. If
         ``False``, they will not be written to the file.

pandas/tests/io/test_parquet.py

@@ -404,12 +404,6 @@ def test_columns_dtypes(self, engine):
 
     @pytest.mark.parametrize("compression", [None, "gzip", "snappy", "brotli"])
     def test_compression(self, engine, compression):
-        if compression == "snappy":
-            pytest.importorskip("snappy")
-
-        elif compression == "brotli":
-            pytest.importorskip("brotli")
-
         df = pd.DataFrame({"A": [1, 2, 3]})
         check_round_trip(df, engine, write_kwargs={"compression": compression})
 

pyproject.toml

@@ -81,13 +81,12 @@ xml = ['lxml>=4.8.0']
 plot = ['matplotlib>=3.6.1']
 output_formatting = ['jinja2>=3.1.2', 'tabulate>=0.8.10']
 clipboard = ['PyQt5>=5.15.6', 'qtpy>=2.2.0']
-compression = ['brotlipy>=0.7.0', 'python-snappy>=0.6.1', 'zstandard>=0.17.0']
+compression = ['zstandard>=0.17.0']
 consortium-standard = ['dataframe-api-compat>=0.1.7']
 all = ['beautifulsoup4>=4.11.1',
        # blosc only available on conda (https://github.com/Blosc/python-blosc/issues/297)
        #'blosc>=1.21.0',
        'bottleneck>=1.3.4',
-       'brotlipy>=0.7.0',
        'dataframe-api-compat>=0.1.7',
        'fastparquet>=0.8.1',
        'fsspec>=2022.05.0',
@@ -110,7 +109,6 @@ all = ['beautifulsoup4>=4.11.1',
        'pytest>=7.3.2',
        'pytest-xdist>=2.2.0',
        'pytest-asyncio>=0.17.0',
-       'python-snappy>=0.6.1',
        'pyxlsb>=1.0.9',
        'qtpy>=2.2.0',
        'scipy>=1.8.1',

requirements-dev.txt

@@ -16,7 +16,6 @@ numpy
 pytz
 beautifulsoup4>=4.11.1
 blosc
-brotlipy>=0.7.0
 bottleneck>=1.3.4
 fastparquet>=0.8.1
 fsspec>=2022.05.0
@@ -37,7 +36,6 @@ pyarrow>=7.0.0
 pymysql>=1.0.2
 pyreadstat>=1.1.5
 tables>=3.7.0
-python-snappy>=0.6.1
 pyxlsb>=1.0.9
 s3fs>=2022.05.0
 scipy>=1.8.1

scripts/tests/data/deps_expected_random.yaml

@@ -26,7 +26,6 @@ dependencies:
   - beautifulsoup4>=5.9.3
   - blosc
   - bottleneck>=1.3.2
-  - brotlipy>=0.7.0
   - fastparquet>=0.6.3
   - fsspec>=2021.07.0
   - html5lib>=1.1
@@ -45,7 +44,6 @@ dependencies:
   - pymysql>=1.0.2
   - pyreadstat>=1.1.2
   - pytables>=3.6.1
-  - python-snappy>=0.6.0
   - pyxlsb>=1.0.8
   - s3fs>=2021.08.0
   - scipy>=1.7.1

scripts/tests/data/deps_minimum.toml

@@ -77,12 +77,11 @@ xml = ['lxml>=4.6.3']
 plot = ['matplotlib>=3.6.1']
 output_formatting = ['jinja2>=3.0.0', 'tabulate>=0.8.9']
 clipboard = ['PyQt5>=5.15.1', 'qtpy>=2.2.0']
-compression = ['brotlipy>=0.7.0', 'python-snappy>=0.6.0', 'zstandard>=0.15.2']
+compression = ['zstandard>=0.15.2']
 all = ['beautifulsoup4>=5.9.3',
        # blosc only available on conda (https://github.com/Blosc/python-blosc/issues/297)
        #'blosc>=1.21.0',
        'bottleneck>=1.3.2',
-       'brotlipy>=0.7.0',
        'fastparquet>=0.6.3',
        'fsspec>=2021.07.0',
        'gcsfs>=2021.07.0',
@@ -104,7 +103,6 @@ all = ['beautifulsoup4>=5.9.3',
        'pytest>=7.3.2',
        'pytest-xdist>=2.2.0',
        'pytest-asyncio>=0.17.0',
-       'python-snappy>=0.6.0',
        'pyxlsb>=1.0.8',
        'qtpy>=2.2.0',
        'scipy>=1.7.1',

scripts/tests/data/deps_unmodified_random.yaml

@@ -26,7 +26,6 @@ dependencies:
   - beautifulsoup4
   - blosc
   - bottleneck>=1.3.2
-  - brotlipy
   - fastparquet>=0.6.3
   - fsspec>=2021.07.0
   - html5lib>=1.1
@@ -45,7 +44,6 @@ dependencies:
   - pymysql>=1.0.2
   - pyreadstat>=1.1.2
   - pytables>=3.6.1
-  - python-snappy>=0.6.0
   - pyxlsb>=1.0.8
   - s3fs>=2021.08.0
   - scipy>=1.7.1