Backport of CI related PRs on 1.4.x (pandas-dev#46559)

Author: simonjayhawkins

.circleci/config.yml

@@ -8,7 +8,7 @@ jobs:
     environment:
       ENV_FILE: ci/deps/circle-38-arm64.yaml
       PYTEST_WORKERS: auto
-      PATTERN: "not slow and not network and not clipboard and not arm_slow"
+      PATTERN: "not single_cpu and not slow and not network and not clipboard and not arm_slow and not db"
       PYTEST_TARGET: "pandas"
     steps:
       - checkout

.github/workflows/posix.yml

@@ -162,8 +162,7 @@ jobs:
       shell: bash
       run: |
         # TODO: re-enable cov, its slowing the tests down though
-        # TODO: Unpin Cython, the new Cython 0.29.26 is causing compilation errors
-        pip install Cython==0.29.25 numpy python-dateutil pytz pytest>=6.0 pytest-xdist>=1.31.0 hypothesis>=5.5.3
+        pip install Cython numpy python-dateutil pytz pytest>=6.0 pytest-xdist>=1.31.0 pytest-asyncio>=0.17 hypothesis>=5.5.3
       if: ${{ env.IS_PYPY == 'true' }}
 
     - name: Build Pandas

azure-pipelines.yml

@@ -45,7 +45,7 @@ jobs:
         /opt/python/cp38-cp38/bin/python -m venv ~/virtualenvs/pandas-dev && \
         . ~/virtualenvs/pandas-dev/bin/activate && \
         python -m pip install --no-deps -U pip wheel 'setuptools<60.0.0' && \
-        pip install cython numpy python-dateutil pytz pytest pytest-xdist hypothesis pytest-azurepipelines && \
+        pip install cython numpy python-dateutil pytz pytest pytest-xdist pytest-asyncio>=0.17 hypothesis && \
         python setup.py build_ext -q -j2 && \
         python -m pip install --no-build-isolation -e . && \
         pytest -m 'not slow and not network and not clipboard' pandas --junitxml=test-data.xml"

ci/deps/actions-310-numpydev.yaml

@@ -9,6 +9,7 @@ dependencies:
   - pytest-cov
   - pytest-xdist>=1.31
   - hypothesis>=5.5.3
+  - pytest-asyncio>=0.17
 
   # pandas dependencies
   - python-dateutil

ci/deps/actions-310.yaml

@@ -11,6 +11,8 @@ dependencies:
   - pytest-xdist>=1.31
   - hypothesis>=5.5.3
   - psutil
+  - pytest-asyncio>=0.17
+  - boto3
 
   # required dependencies
   - python-dateutil
@@ -21,6 +23,7 @@ dependencies:
   - beautifulsoup4
   - blosc
   - bottleneck
+  - brotlipy
   - fastparquet
   - fsspec
   - html5lib
@@ -39,6 +42,7 @@ dependencies:
   - pytables
   - pyarrow
   - pyreadstat
+  - python-snappy
   - pyxlsb
   - s3fs
   - scipy

ci/deps/actions-38-downstream_compat.yaml

@@ -12,6 +12,8 @@ dependencies:
   - pytest-xdist>=1.31
   - hypothesis>=5.5.3
   - psutil
+  - pytest-asyncio>=0.17
+  - boto3
 
   # required dependencies
   - python-dateutil
@@ -21,6 +23,7 @@ dependencies:
   # optional dependencies
   - beautifulsoup4
   - blosc
+  - brotlipy
   - bottleneck
   - fastparquet
   - fsspec
@@ -35,10 +38,11 @@ dependencies:
   - odfpy
   - pandas-gbq
   - psycopg2
-  - pymysql
-  - pytables
   - pyarrow
+  - pymysql
   - pyreadstat
+  - pytables
+  - python-snappy
   - pyxlsb
   - s3fs
   - scipy
@@ -52,17 +56,14 @@ dependencies:
 
   # downstream packages
   - aiobotocore
-  - boto3
   - botocore
   - cftime
   - dask
   - ipython
   - geopandas
-  - python-snappy
   - seaborn
   - scikit-learn
   - statsmodels
-  - brotlipy
   - coverage
   - pandas-datareader
   - pyyaml

ci/deps/actions-38-minimum_versions.yaml

@@ -13,6 +13,8 @@ dependencies:
   - pytest-xdist>=1.31
   - hypothesis>=5.5.3
   - psutil
+  - pytest-asyncio>=0.17
+  - boto3
 
   # required dependencies
   - python-dateutil=2.8.1
@@ -23,6 +25,7 @@ dependencies:
   - beautifulsoup4=4.8.2
   - blosc=1.20.1
   - bottleneck=1.3.1
+  - brotlipy=0.7.0
   - fastparquet=0.4.0
   - fsspec=0.7.4
   - html5lib=1.1
@@ -37,10 +40,11 @@ dependencies:
   - openpyxl=3.0.3
   - pandas-gbq=0.14.0
   - psycopg2=2.8.4
-  - pymysql=0.10.1
-  - pytables=3.6.1
   - pyarrow=1.0.1
+  - pymysql=0.10.1
   - pyreadstat=1.1.0
+  - pytables=3.6.1
+  - python-snappy=0.6.0
   - pyxlsb=1.0.6
   - s3fs=0.4.0
   - scipy=1.4.1

ci/deps/actions-38.yaml

@@ -11,6 +11,8 @@ dependencies:
   - pytest-xdist>=1.31
   - hypothesis>=5.5.3
   - psutil
+  - pytest-asyncio>=0.17
+  - boto3
 
   # required dependencies
   - python-dateutil
@@ -21,6 +23,7 @@ dependencies:
   - beautifulsoup4
   - blosc
   - bottleneck
+  - brotlipy
   - fastparquet
   - fsspec
   - html5lib
@@ -34,10 +37,11 @@ dependencies:
   - odfpy
   - pandas-gbq
   - psycopg2
-  - pymysql
-  - pytables
   - pyarrow
+  - pymysql
   - pyreadstat
+  - pytables
+  - python-snappy
   - pyxlsb
   - s3fs
   - scipy

ci/deps/actions-39.yaml

@@ -11,6 +11,8 @@ dependencies:
   - pytest-xdist>=1.31
   - hypothesis>=5.5.3
   - psutil
+  - pytest-asyncio>=0.17
+  - boto3
 
   # required dependencies
   - python-dateutil
@@ -21,6 +23,7 @@ dependencies:
   - beautifulsoup4
   - blosc
   - bottleneck
+  - brotlipy
   - fastparquet
   - fsspec
   - html5lib
@@ -35,9 +38,10 @@ dependencies:
   - pandas-gbq
   - psycopg2
   - pymysql
-  - pytables
   - pyarrow
   - pyreadstat
+  - pytables
+  - python-snappy
   - pyxlsb
   - s3fs
   - scipy

ci/deps/circle-38-arm64.yaml

@@ -4,18 +4,52 @@ channels:
 dependencies:
   - python=3.8
 
-  # tools
-  - cython>=0.29.24
+  # test dependencies
+  - cython=0.29.24
   - pytest>=6.0
+  - pytest-cov
   - pytest-xdist>=1.31
   - hypothesis>=5.5.3
+  - psutil
+  - pytest-asyncio>=0.17
+  - boto3
 
-  # pandas dependencies
-  - botocore>=1.11
-  - flask
-  - moto
-  - numpy
+  # required dependencies
   - python-dateutil
+  - numpy
   - pytz
+
+  # optional dependencies
+  - beautifulsoup4
+  - blosc
+  - bottleneck
+  - brotlipy
+  - fastparquet
+  - fsspec
+  - html5lib
+  - gcsfs
+  - jinja2
+  - lxml
+  - matplotlib
+  - numba
+  - numexpr
+  - openpyxl
+  - odfpy
+  - pandas-gbq
+  - psycopg2
+  - pyarrow
+  - pymysql
+  # Not provided on ARM
+  #- pyreadstat
+  - pytables
+  - python-snappy
+  - pyxlsb
+  - s3fs
+  - scipy
+  - sqlalchemy
+  - tabulate
+  - xarray
+  - xlrd
+  - xlsxwriter
+  - xlwt
   - zstandard
-  - pip

doc/source/getting_started/install.rst

@@ -410,5 +410,7 @@ Compression
 ========================= ================== =============================================================
 Dependency                Minimum Version    Notes
 ========================= ================== =============================================================
-Zstandard                                    Zstandard compression
+brotli                    0.7.0              Brotli compression
+python-snappy             0.6.0              Snappy compression
+Zstandard                 0.15.2             Zstandard compression
 ========================= ================== =============================================================

environment.yml

@@ -69,7 +69,7 @@ dependencies:
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31
-  - pytest-asyncio
+  - pytest-asyncio>=0.17
   - pytest-instafail
 
   # downstream tests

pandas/compat/_optional.py

@@ -13,6 +13,7 @@
     "bs4": "4.8.2",
     "blosc": "1.20.1",
     "bottleneck": "1.3.1",
+    "brotli": "0.7.0",
     "fastparquet": "0.4.0",
     "fsspec": "0.7.4",
     "html5lib": "1.1",
@@ -34,6 +35,7 @@
     "pyxlsb": "1.0.6",
     "s3fs": "0.4.0",
     "scipy": "1.4.1",
+    "snappy": "0.6.0",
     "sqlalchemy": "1.4.0",
     "tables": "3.6.1",
     "tabulate": "0.8.7",
@@ -50,12 +52,14 @@
 INSTALL_MAPPING = {
     "bs4": "beautifulsoup4",
     "bottleneck": "Bottleneck",
+    "brotli": "brotlipy",
+    "jinja2": "Jinja2",
     "lxml.etree": "lxml",
     "odf": "odfpy",
     "pandas_gbq": "pandas-gbq",
-    "tables": "pytables",
+    "snappy": "python-snappy",
     "sqlalchemy": "SQLAlchemy",
-    "jinja2": "Jinja2",
+    "tables": "pytables",
 }
 
 
@@ -66,6 +70,13 @@ def get_version(module: types.ModuleType) -> str:
         version = getattr(module, "__VERSION__", None)
 
     if version is None:
+        if module.__name__ == "brotli":
+            # brotli doesn't contain attributes to confirm it's version
+            return ""
+        if module.__name__ == "snappy":
+            # snappy doesn't contain attributes to confirm it's version
+            # See https://github.com/andrix/python-snappy/pull/119
+            return ""
         raise ImportError(f"Can't determine version for {module.__name__}")
     if module.__name__ == "psycopg2":
         # psycopg2 appends " (dt dec pq3 ext lo64)" to it's version
@@ -141,7 +152,7 @@ def import_optional_dependency(
     minimum_version = min_version if min_version is not None else VERSIONS.get(parent)
     if minimum_version:
         version = get_version(module_to_get)
-        if Version(version) < Version(minimum_version):
+        if version and Version(version) < Version(minimum_version):
             msg = (
                 f"Pandas requires version '{minimum_version}' or newer of '{parent}' "
                 f"(version '{version}' currently installed)."

pandas/tests/arrays/string_/test_string.py

@@ -504,7 +504,7 @@ def test_memory_usage(dtype):
     # GH 33963
 
     if dtype.storage == "pyarrow":
-        pytest.skip("not applicable")
+        pytest.skip(f"not applicable for {dtype.storage}")
 
     series = pd.Series(["a", "b", "c"], dtype=dtype)
 

pandas/tests/extension/arrow/test_bool.py

@@ -1,6 +1,11 @@
 import numpy as np
 import pytest
 
+from pandas.compat import (
+    is_ci_environment,
+    is_platform_windows,
+)
+
 import pandas as pd
 import pandas._testing as tm
 from pandas.api.types import is_bool_dtype
@@ -91,6 +96,10 @@ def test_reduce_series_boolean(self):
         pass
 
 
+@pytest.mark.skipif(
+    is_ci_environment() and is_platform_windows(),
+    reason="Causes stack overflow on Windows CI",
+)
 class TestReduceBoolean(base.BaseBooleanReduceTests):
     pass
 

pandas/tests/extension/base/ops.py

@@ -114,17 +114,22 @@ def test_add_series_with_extension_array(self, data):
         self.assert_series_equal(result, expected)
 
     @pytest.mark.parametrize("box", [pd.Series, pd.DataFrame])
-    def test_direct_arith_with_ndframe_returns_not_implemented(self, data, box):
+    def test_direct_arith_with_ndframe_returns_not_implemented(
+        self, request, data, box
+    ):
         # EAs should return NotImplemented for ops with Series/DataFrame
         # Pandas takes care of unboxing the series and calling the EA's op.
         other = pd.Series(data)
         if box is pd.DataFrame:
             other = other.to_frame()
-        if hasattr(data, "__add__"):
-            result = data.__add__(other)
-            assert result is NotImplemented
-        else:
-            raise pytest.skip(f"{type(data).__name__} does not implement add")
+        if not hasattr(data, "__add__"):
+            request.node.add_marker(
+                pytest.mark.xfail(
+                    reason=f"{type(data).__name__} does not implement add"
+                )
+            )
+        result = data.__add__(other)
+        assert result is NotImplemented
 
 
 class BaseComparisonOpsTests(BaseOpsUtil):