CI: Ensure all minimum optional dependencies are tested (#45103)

Author: mroeschke

.github/workflows/posix.yml

@@ -25,8 +25,8 @@ jobs:
     strategy:
       matrix:
         settings: [
-          [actions-38-db-min.yaml, "((not slow and not network and not clipboard) or (single and db))", "", "", "", "", ""],
-          [actions-38-db.yaml, "((not slow and not network and not clipboard) or (single and db))", "", "", "", "", ""],
+          [actions-38-downstream_compat.yaml, "not slow and not network and not clipboard", "", "", "", "", ""],
+          [actions-38-minimum_versions.yaml, "slow", "", "", "", "", ""],
           [actions-38-minimum_versions.yaml, "not slow and not network and not clipboard", "", "", "", "", ""],
           [actions-38-locale_slow.yaml, "slow", "language-pack-it xsel", "it_IT.utf8", "it_IT.utf8", "", ""],
           [actions-38.yaml, "not slow and not clipboard", "", "", "", "", ""],

ci/deps/actions-38-db-min.yaml

@@ -1,54 +1,66 @@
+# Non-dependencies that pandas utilizes or has compatibility with pandas objects
 name: pandas-dev
 channels:
   - conda-forge
 dependencies:
   - python=3.8
   - pip
 
-  # tools
+  # test dependencies
   - cython>=0.29.24
   - pytest>=6.0
   - pytest-xdist>=1.31
   - hypothesis>=5.5.3
   - pytest-cov>=2.10.1  # this is only needed in the coverage build, ref: GH 35737
+  - nomkl
 
-  # pandas dependencies
-  - aiobotocore<2.0.0  # GH#44311 pinned to fix docbuild
+  # required dependencies
+  - numpy
+  - python-dateutil
+  - pytz
+
+  # optional dependencies
   - beautifulsoup4
-  - boto3
-  - botocore>=1.11
-  - dask
+  - blosc
   - fastparquet>=0.4.0
   - fsspec>=0.7.4
-  - gcsfs>=0.6.0
-  - geopandas
+  - gcsfs
   - html5lib
+  - jinja2
+  - lxml
   - matplotlib
-  - nomkl
   - numexpr
-  - numpy=1.18
   - odfpy
   - openpyxl
   - pandas-gbq
   - psycopg2
   - pyarrow>=1.0.1
   - pymysql
   - pytables
-  - python-snappy
-  - python-dateutil
-  - pytz
+  - pyxlsb
   - s3fs>=0.4.0
-  - scikit-learn
   - scipy
   - sqlalchemy
-  - statsmodels
   - xarray
   - xlrd
   - xlsxwriter
   - xlwt
+
+  # downstream packages
+  - aiobotocore<2.0.0  # GH#44311 pinned to fix docbuild
+  - boto3
+  - botocore>=1.11
+  - dask
+  - ipython
+  - geopandas
+  - python-snappy
+  - seaborn
+  - scikit-learn
+  - statsmodels
   - brotlipy
   - coverage
   - pandas-datareader
-  - pyxlsb
+  - pyyaml
+  - py
   - pip:
     - torch

ci/deps/actions-38-db.yaml renamed to ci/deps/actions-38-downstream_compat.yaml

@@ -19,7 +19,7 @@ dependencies:
   - jinja2
   - jedi
   - lxml
-  - matplotlib<3.3.0
+  - matplotlib
   - nomkl
   - numexpr
   - numpy<1.20  # GH#39541 compat with pyarrow<3

ci/deps/actions-38-locale.yaml

@@ -13,18 +13,18 @@ dependencies:
   - hypothesis>=5.5.3
 
   # pandas dependencies
-  - beautifulsoup4=4.8.2
-  - bottleneck=1.3.1
+  - beautifulsoup4
+  - bottleneck
   - lxml
-  - matplotlib=3.3.2
-  - numpy=1.18
-  - openpyxl=3.0.2
+  - matplotlib
+  - numpy
+  - openpyxl
   - python-dateutil
   - python-blosc
   - pytz=2020.1
   - scipy
-  - sqlalchemy=1.4.0
-  - xlrd=2.0.1
-  - xlsxwriter=1.2.2
-  - xlwt=1.3.0
-  - html5lib=1.1
+  - sqlalchemy
+  - xlrd
+  - xlsxwriter
+  - xlwt
+  - html5lib

ci/deps/actions-38-locale_slow.yaml

@@ -1,31 +1,52 @@
+# Minimum version of required + optional dependencies
+# Aligned with getting_started/install.rst and compat/_optional.py
 name: pandas-dev
 channels:
   - conda-forge
 dependencies:
   - python=3.8.0
 
-  # tools
+  # test dependencies
   - cython=0.29.24
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31
   - hypothesis>=5.5.3
   - psutil
 
-  # pandas dependencies
+  # required dependencies
+  - python-dateutil=2.8.1
+  - numpy=1.18.5
+  - pytz=2020.1
+
+  # optional dependencies
   - beautifulsoup4=4.8.2
+  - blosc=1.20.1
   - bottleneck=1.3.1
+  - fastparquet=0.4.0
+  - fsspec=0.7.4
+  - html5lib=1.1
+  - gcsfs=0.6.0
   - jinja2=2.11
+  - lxml=4.5.0
+  - matplotlib=3.3.2
   - numba=0.50.1
   - numexpr=2.7.1
-  - numpy=1.18.5
-  - openpyxl=3.0.2
+  - openpyxl=3.0.3
+  - odfpy=1.4.1
+  - pandas-gbq=0.14.0
+  - psycopg2=2.8.4
+  - pymysql=0.10.1
   - pytables=3.6.1
-  - python-dateutil=2.8.1
-  - pytz=2020.1
   - pyarrow=1.0.1
+  - pyreadstat
+  - pyxlsb=1.0.6
+  - s3fs=0.4.0
   - scipy=1.4.1
+  - sqlalchemy=1.4.0
+  - tabulate=0.8.7
+  - xarray=0.15.1
   - xlrd=2.0.1
   - xlsxwriter=1.2.2
   - xlwt=1.3.0
-  - html5lib=1.1
+  - zstandard=0.15.2

ci/deps/actions-38-minimum_versions.yaml

@@ -17,4 +17,4 @@ dependencies:
   - python-dateutil
   - nomkl
   - pytz
-  - tabulate==0.8.7
+  - tabulate

ci/deps/actions-38.yaml

@@ -291,7 +291,7 @@ Dependency                Minimum Version    Notes
 xlrd                      2.0.1              Reading Excel
 xlwt                      1.3.0              Writing Excel
 xlsxwriter                1.2.2              Writing Excel
-openpyxl                  3.0.2              Reading / writing for xlsx files
+openpyxl                  3.0.3              Reading / writing for xlsx files
 pyxlsb                    1.0.6              Reading for xlsb files
 ========================= ================== =============================================================
 

doc/source/getting_started/install.rst

@@ -430,7 +430,7 @@ Optional libraries below the lowest tested version may still work, but are not c
 +-----------------+-----------------+---------+
 | numba           | 0.50.1          |    X    |
 +-----------------+-----------------+---------+
-| openpyxl        | 3.0.2           |    X    |
+| openpyxl        | 3.0.3           |    X    |
 +-----------------+-----------------+---------+
 | pyarrow         | 1.0.1           |    X    |
 +-----------------+-----------------+---------+

doc/source/whatsnew/v1.4.0.rst

@@ -19,7 +19,7 @@
     "matplotlib": "3.3.2",
     "numexpr": "2.7.1",
     "odfpy": "1.4.1",
-    "openpyxl": "3.0.2",
+    "openpyxl": "3.0.3",
     "pandas_gbq": "0.14.0",
     "pyarrow": "1.0.1",
     "pytest": "6.0",

pandas/compat/_optional.py

@@ -3,6 +3,8 @@
 import numpy as np
 import pytest
 
+from pandas.compat._optional import VERSIONS
+
 from pandas import (
     DataFrame,
     date_range,
@@ -289,7 +291,21 @@ def test_stata_options(fsspectest):
 
 
 @td.skip_if_no("tabulate")
-def test_markdown_options(fsspectest):
+def test_markdown_options(request, fsspectest):
+    import fsspec
+
+    # error: Library stubs not installed for "tabulate"
+    # (or incompatible with Python 3.8)
+    import tabulate  # type: ignore[import]
+
+    request.node.add_marker(
+        pytest.mark.xfail(
+            fsspec.__version__ == VERSIONS["fsspec"]
+            and tabulate.__version__ == VERSIONS["tabulate"],
+            reason="Fails on the min version build",
+            raises=FileNotFoundError,
+        )
+    )
     df = DataFrame({"a": [0]})
     df.to_markdown("testmem://afile", storage_options={"test": "md_write"})
     assert fsspectest.test[0] == "md_write"

pandas/tests/io/test_fsspec.py

@@ -171,10 +171,6 @@ def test_reindex_nearest():
     tm.assert_series_equal(expected, result)
 
 
-def test_reindex_backfill():
-    pass
-
-
 def test_reindex_int(datetime_series):
     ts = datetime_series[::2]
     int_ts = Series(np.zeros(len(ts), dtype=int), index=ts.index)