From de340b7d802a497eda17b401a84a749254c077fa Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Mon, 7 Mar 2022 17:49:25 -0800 Subject: [PATCH 1/6] CI: Add missing dependency for skipped tests --- ci/deps/actions-310.yaml | 4 ++++ ci/deps/actions-38-downstream_compat.yaml | 7 ++++--- ci/deps/actions-38-minimum_versions.yaml | 4 ++++ ci/deps/actions-38.yaml | 4 ++++ ci/deps/actions-39.yaml | 4 ++++ doc/source/getting_started/install.rst | 6 +++++- pandas/compat/_optional.py | 3 +++ 7 files changed, 28 insertions(+), 4 deletions(-) diff --git a/ci/deps/actions-310.yaml b/ci/deps/actions-310.yaml index bbc468f9d8f43..8f333c72f002c 100644 --- a/ci/deps/actions-310.yaml +++ b/ci/deps/actions-310.yaml @@ -11,6 +11,8 @@ dependencies: - pytest-xdist>=1.31 - hypothesis>=5.5.3 - psutil + - pytest-asyncio + - boto3 # required dependencies - python-dateutil @@ -21,6 +23,7 @@ dependencies: - beautifulsoup4 - blosc - bottleneck + - brotlipy - fastparquet - fsspec - html5lib @@ -42,6 +45,7 @@ dependencies: - pyxlsb - s3fs - scipy + - snappy - sqlalchemy - tabulate - xarray diff --git a/ci/deps/actions-38-downstream_compat.yaml b/ci/deps/actions-38-downstream_compat.yaml index f3fa95d03c98e..7386649f1c627 100644 --- a/ci/deps/actions-38-downstream_compat.yaml +++ b/ci/deps/actions-38-downstream_compat.yaml @@ -12,6 +12,8 @@ dependencies: - pytest-xdist>=1.31 - hypothesis>=5.5.3 - psutil + - pytest-asyncio + - boto3 # required dependencies - python-dateutil @@ -21,6 +23,7 @@ dependencies: # optional dependencies - beautifulsoup4 - blosc + - brotlipy - bottleneck - fastparquet - fsspec @@ -42,6 +45,7 @@ dependencies: - pyxlsb - s3fs - scipy + - snappy - sqlalchemy - tabulate - xarray @@ -52,17 +56,14 @@ dependencies: # downstream packages - aiobotocore - - boto3 - botocore - cftime - dask - ipython - geopandas - - python-snappy - seaborn - scikit-learn - statsmodels - - brotlipy - coverage - pandas-datareader - pyyaml diff --git a/ci/deps/actions-38-minimum_versions.yaml b/ci/deps/actions-38-minimum_versions.yaml index 2e782817f3f14..5726001ea1290 100644 --- a/ci/deps/actions-38-minimum_versions.yaml +++ b/ci/deps/actions-38-minimum_versions.yaml @@ -13,6 +13,8 @@ dependencies: - pytest-xdist>=1.31 - hypothesis>=5.5.3 - psutil + - pytest-asyncio + - boto3 # required dependencies - python-dateutil=2.8.1 @@ -23,6 +25,7 @@ dependencies: - beautifulsoup4=4.8.2 - blosc=1.20.1 - bottleneck=1.3.1 + - brotlipy=0.7.0 - fastparquet=0.4.0 - fsspec=0.7.4 - html5lib=1.1 @@ -44,6 +47,7 @@ dependencies: - pyxlsb=1.0.6 - s3fs=0.4.0 - scipy=1.4.1 + - snappy=1.1.8 - sqlalchemy=1.4.0 - tabulate=0.8.7 - xarray=0.15.1 diff --git a/ci/deps/actions-38.yaml b/ci/deps/actions-38.yaml index 60db02def8a3d..85a25571bb9fd 100644 --- a/ci/deps/actions-38.yaml +++ b/ci/deps/actions-38.yaml @@ -11,6 +11,8 @@ dependencies: - pytest-xdist>=1.31 - hypothesis>=5.5.3 - psutil + - pytest-asyncio + - boto3 # required dependencies - python-dateutil @@ -21,6 +23,7 @@ dependencies: - beautifulsoup4 - blosc - bottleneck + - brotlipy - fastparquet - fsspec - html5lib @@ -41,6 +44,7 @@ dependencies: - pyxlsb - s3fs - scipy + - snappy - sqlalchemy - tabulate - xarray diff --git a/ci/deps/actions-39.yaml b/ci/deps/actions-39.yaml index 2d6430afd0b36..8cff9539300f0 100644 --- a/ci/deps/actions-39.yaml +++ b/ci/deps/actions-39.yaml @@ -11,6 +11,8 @@ dependencies: - pytest-xdist>=1.31 - hypothesis>=5.5.3 - psutil + - pytest-asyncio + - boto3 # required dependencies - python-dateutil @@ -21,6 +23,7 @@ dependencies: - beautifulsoup4 - blosc - bottleneck + - brotlipy - fastparquet - fsspec - html5lib @@ -41,6 +44,7 @@ dependencies: - pyxlsb - s3fs - scipy + - snappy - sqlalchemy - tabulate - xarray diff --git a/doc/source/getting_started/install.rst b/doc/source/getting_started/install.rst index 8fe0fd1f78e61..b99f6843a6f92 100644 --- a/doc/source/getting_started/install.rst +++ b/doc/source/getting_started/install.rst @@ -410,5 +410,9 @@ Compression ========================= ================== ============================================================= Dependency Minimum Version Notes ========================= ================== ============================================================= -Zstandard Zstandard compression +brotli 0.7.0 Brotli compression +========================= ================== ============================================================= +snappy 1.1.8 Snappy compression +========================= ================== ============================================================= +Zstandard 0.15.2 Zstandard compression ========================= ================== ============================================================= diff --git a/pandas/compat/_optional.py b/pandas/compat/_optional.py index cb9b0c6de6d3b..76c5c1784f472 100644 --- a/pandas/compat/_optional.py +++ b/pandas/compat/_optional.py @@ -13,6 +13,7 @@ "bs4": "4.8.2", "blosc": "1.20.1", "bottleneck": "1.3.1", + "brotli": "0.7.0", "fastparquet": "0.4.0", "fsspec": "0.7.4", "html5lib": "1.1", @@ -34,6 +35,7 @@ "pyxlsb": "1.0.6", "s3fs": "0.4.0", "scipy": "1.4.1", + "snappy": "1.1.8", "sqlalchemy": "1.4.0", "tables": "3.6.1", "tabulate": "0.8.7", @@ -50,6 +52,7 @@ INSTALL_MAPPING = { "bs4": "beautifulsoup4", "bottleneck": "Bottleneck", + "brotli": "brotlipy", "lxml.etree": "lxml", "odf": "odfpy", "pandas_gbq": "pandas-gbq", From f9f120c2e03eb8356bd24b98e7e0c41c9bc98847 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Mon, 7 Mar 2022 18:26:51 -0800 Subject: [PATCH 2/6] Make exception for brotili --- pandas/compat/_optional.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/compat/_optional.py b/pandas/compat/_optional.py index 76c5c1784f472..5fa90fa15dc57 100644 --- a/pandas/compat/_optional.py +++ b/pandas/compat/_optional.py @@ -142,7 +142,8 @@ def import_optional_dependency( else: module_to_get = module minimum_version = min_version if min_version is not None else VERSIONS.get(parent) - if minimum_version: + # brotli doesn't contain module info to confirm it's version + if minimum_version and module_to_get != "brotli": version = get_version(module_to_get) if Version(version) < Version(minimum_version): msg = ( From 4fdd58cedbb06756a3d29a20bd9fa3e6247ea8d6 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Mon, 7 Mar 2022 20:16:58 -0800 Subject: [PATCH 3/6] Check name instead --- pandas/compat/_optional.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/compat/_optional.py b/pandas/compat/_optional.py index 5fa90fa15dc57..2b8062565d0d6 100644 --- a/pandas/compat/_optional.py +++ b/pandas/compat/_optional.py @@ -143,7 +143,7 @@ def import_optional_dependency( module_to_get = module minimum_version = min_version if min_version is not None else VERSIONS.get(parent) # brotli doesn't contain module info to confirm it's version - if minimum_version and module_to_get != "brotli": + if minimum_version and name != "brotli": version = get_version(module_to_get) if Version(version) < Version(minimum_version): msg = ( From cc4075f02812b7b837ce1e895f14d63577e0d2ee Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Mon, 7 Mar 2022 20:50:52 -0800 Subject: [PATCH 4/6] Consolidate brotli version logic --- pandas/compat/_optional.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pandas/compat/_optional.py b/pandas/compat/_optional.py index 2b8062565d0d6..de48db37cc67a 100644 --- a/pandas/compat/_optional.py +++ b/pandas/compat/_optional.py @@ -62,13 +62,16 @@ } -def get_version(module: types.ModuleType) -> str: +def get_version(module: types.ModuleType) -> str | None: version = getattr(module, "__version__", None) if version is None: # xlrd uses a capitalized attribute name version = getattr(module, "__VERSION__", None) if version is None: + if module.__name__ == "brotli": + # brotli doesn't contain attributes to confirm it's version + return None raise ImportError(f"Can't determine version for {module.__name__}") if module.__name__ == "psycopg2": # psycopg2 appends " (dt dec pq3 ext lo64)" to it's version @@ -142,10 +145,9 @@ def import_optional_dependency( else: module_to_get = module minimum_version = min_version if min_version is not None else VERSIONS.get(parent) - # brotli doesn't contain module info to confirm it's version - if minimum_version and name != "brotli": + if minimum_version: version = get_version(module_to_get) - if Version(version) < Version(minimum_version): + if version is not None and Version(version) < Version(minimum_version): msg = ( f"Pandas requires version '{minimum_version}' or newer of '{parent}' " f"(version '{version}' currently installed)." From f73cd9aa5d39081d6baec2e6f5ef05db26e47faa Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Mon, 7 Mar 2022 21:36:52 -0800 Subject: [PATCH 5/6] Fix typing --- pandas/compat/_optional.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pandas/compat/_optional.py b/pandas/compat/_optional.py index de48db37cc67a..baae5c913c05d 100644 --- a/pandas/compat/_optional.py +++ b/pandas/compat/_optional.py @@ -62,7 +62,7 @@ } -def get_version(module: types.ModuleType) -> str | None: +def get_version(module: types.ModuleType) -> str: version = getattr(module, "__version__", None) if version is None: # xlrd uses a capitalized attribute name @@ -71,7 +71,7 @@ def get_version(module: types.ModuleType) -> str | None: if version is None: if module.__name__ == "brotli": # brotli doesn't contain attributes to confirm it's version - return None + return "" raise ImportError(f"Can't determine version for {module.__name__}") if module.__name__ == "psycopg2": # psycopg2 appends " (dt dec pq3 ext lo64)" to it's version @@ -147,7 +147,7 @@ def import_optional_dependency( minimum_version = min_version if min_version is not None else VERSIONS.get(parent) if minimum_version: version = get_version(module_to_get) - if version is not None and Version(version) < Version(minimum_version): + if version and Version(version) < Version(minimum_version): msg = ( f"Pandas requires version '{minimum_version}' or newer of '{parent}' " f"(version '{version}' currently installed)." From 415793a247c1082a7bb18bb5c0c01ba729216c0f Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Tue, 8 Mar 2022 09:50:53 -0800 Subject: [PATCH 6/6] reformat table --- doc/source/getting_started/install.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/doc/source/getting_started/install.rst b/doc/source/getting_started/install.rst index b99f6843a6f92..8706292d60def 100644 --- a/doc/source/getting_started/install.rst +++ b/doc/source/getting_started/install.rst @@ -411,8 +411,6 @@ Compression Dependency Minimum Version Notes ========================= ================== ============================================================= brotli 0.7.0 Brotli compression -========================= ================== ============================================================= snappy 1.1.8 Snappy compression -========================= ================== ============================================================= Zstandard 0.15.2 Zstandard compression ========================= ================== =============================================================