CI, ENH: Check each minimum dependency is enforced in *.yaml and environment.yml (#51189)

* Add pin_min_versions_to_ci_deps()

* Fix typos

* Remove debug statements

* Update YAML package versions

* Replace files with updated min versions

* Add exclusions + edge cases

* Revert file to original

* Revert environment.yml

* Fix bugs

* Add helper function

* Ran pre-commit run validate-min-versions-in-sync --all-files

* Fix merge issue

* add parameter for custom toml dependency

* Add dummy test files for test_validate_min_versions_in_sync.py

* Add unit test: test_update_yaml_file_version()

* Add dummy test files for edge cases

* Add parametrization to unit test

* Run pre-commit

* Fix tzdata bug

* Fix tzdata issue

* Refactor

* Update ci deps

* Remove pyarrow

* Revert tzdata

* Update pyarrow

* Revert pyarrow

* Add docstrings

* Remove psutil

* Fix file rewrite issue

* Refactor test data

* minor fixups

* fixup

* improve typing

---------

Co-authored-by: MarcoGorelli <>
Co-authored-by: Marco Edward Gorelli <[email protected]>

Author: kathleenhang

.pre-commit-config.yaml

@@ -313,7 +313,7 @@ repos:
         entry: python scripts/generate_pip_deps_from_conda.py
         files: ^(environment.yml|requirements-dev.txt)$
         pass_filenames: false
-        additional_dependencies: [pyyaml, toml]
+        additional_dependencies: [tomli, pyyaml]
     -   id: title-capitalization
         name: Validate correct capitalization among titles in documentation
         entry: python scripts/validate_rst_title_capitalization.py
@@ -391,10 +391,11 @@ repos:
         types: [yaml]
     -   id: validate-min-versions-in-sync
         name: Check minimum version of dependencies are aligned
-        entry: python scripts/validate_min_versions_in_sync.py
+        entry: python -m scripts.validate_min_versions_in_sync
         language: python
         files: ^(ci/deps/actions-.*-minimum_versions\.yaml|pandas/compat/_optional\.py)$
-        additional_dependencies: [tomli]
+        additional_dependencies: [tomli, pyyaml]
+        pass_filenames: false
     -   id: validate-errors-locations
         name: Validate errors locations
         description: Validate errors are in appropriate locations.

ci/deps/actions-310-numpydev.yaml

@@ -12,7 +12,7 @@ dependencies:
   - pytest-cov
   - pytest-xdist>=2.2.0
   - hypothesis>=6.34.2
-  - pytest-asyncio>=0.17
+  - pytest-asyncio>=0.17.0
 
   # pandas dependencies
   - python-dateutil

ci/deps/actions-310.yaml

@@ -12,7 +12,7 @@ dependencies:
   - pytest>=7.0.0
   - pytest-cov
   - pytest-xdist>=2.2.0
-  - pytest-asyncio>=0.17
+  - pytest-asyncio>=0.17.0
   - boto3
 
   # required dependencies
@@ -21,36 +21,36 @@ dependencies:
   - pytz
 
   # optional dependencies
-  - beautifulsoup4
+  - beautifulsoup4>=4.9.3
   - blosc
-  - bottleneck
-  - brotlipy
-  - fastparquet
-  - fsspec
-  - html5lib
-  - hypothesis
-  - gcsfs
-  - jinja2
-  - lxml
+  - bottleneck>=1.3.2
+  - brotlipy>=0.7.0
+  - fastparquet>=0.6.3
+  - fsspec>=2021.07.0
+  - html5lib>=1.1
+  - hypothesis>=6.34.2
+  - gcsfs>=2021.07.0
+  - jinja2>=3.0.0
+  - lxml>=4.6.3
   - matplotlib>=3.6.1, <3.7.0
-  - numba
-  - numexpr
-  - openpyxl<3.1.1
-  - odfpy
-  - pandas-gbq
-  - psycopg2
-  - pymysql
-  - pytables
+  - numba>=0.53.1
+  - numexpr>=2.7.3
+  - openpyxl<3.1.1, >=3.0.7
+  - odfpy>=1.4.1
+  - pandas-gbq>=0.15.0
+  - psycopg2>=2.8.6
+  - pymysql>=1.0.2
+  - pytables>=3.6.1
   - pyarrow
-  - pyreadstat
-  - python-snappy
-  - pyxlsb
+  - pyreadstat>=1.1.2
+  - python-snappy>=0.6.0
+  - pyxlsb>=1.0.8
   - s3fs>=2021.08.0
-  - scipy
-  - sqlalchemy
-  - tabulate
+  - scipy>=1.7.1
+  - sqlalchemy>=1.4.16
+  - tabulate>=0.8.9
   - tzdata>=2022a
-  - xarray
-  - xlrd
-  - xlsxwriter
-  - zstandard
+  - xarray>=0.21.0
+  - xlrd>=2.0.1
+  - xlsxwriter>=1.4.3
+  - zstandard>=0.15.2

ci/deps/actions-311.yaml

@@ -9,10 +9,10 @@ dependencies:
   - cython>=0.29.32
 
   # test dependencies
-  - pytest>=7.0
+  - pytest>=7.0.0
   - pytest-cov
   - pytest-xdist>=2.2.0
-  - pytest-asyncio>=0.17
+  - pytest-asyncio>=0.17.0
   - boto3
 
   # required dependencies
@@ -21,36 +21,36 @@ dependencies:
   - pytz
 
   # optional dependencies
-  - beautifulsoup4
+  - beautifulsoup4>=4.9.3
   - blosc
-  - bottleneck
-  - brotlipy
-  - fastparquet
-  - fsspec
-  - html5lib
-  - hypothesis
-  - gcsfs
-  - jinja2
-  - lxml
+  - bottleneck>=1.3.2
+  - brotlipy>=0.7.0
+  - fastparquet>=0.6.3
+  - fsspec>=2021.07.0
+  - html5lib>=1.1
+  - hypothesis>=6.34.2
+  - gcsfs>=2021.07.0
+  - jinja2>=3.0.0
+  - lxml>=4.6.3
   - matplotlib>=3.6.1, <3.7.0
   # - numba  not compatible with 3.11
-  - numexpr
-  - openpyxl<3.1.1
-  - odfpy
-  - pandas-gbq
-  - psycopg2
-  - pymysql
+  - numexpr>=2.7.3
+  - openpyxl<3.1.1, >=3.0.7
+  - odfpy>=1.4.1
+  - pandas-gbq>=0.15.0
+  - psycopg2>=2.8.6
+  - pymysql>=1.0.2
   # - pytables>=3.8.0  # first version that supports 3.11
   - pyarrow
-  - pyreadstat
-  - python-snappy
-  - pyxlsb
+  - pyreadstat>=1.1.2
+  - python-snappy>=0.6.0
+  - pyxlsb>=1.0.8
   - s3fs>=2021.08.0
-  - scipy
-  - sqlalchemy
-  - tabulate
+  - scipy>=1.7.1
+  - sqlalchemy>=1.4.16
+  - tabulate>=0.8.9
   - tzdata>=2022a
-  - xarray
-  - xlrd
-  - xlsxwriter
-  - zstandard
+  - xarray>=0.21.0
+  - xlrd>=2.0.1
+  - xlsxwriter>=1.4.3
+  - zstandard>=0.15.2

ci/deps/actions-38-downstream_compat.yaml

@@ -13,7 +13,7 @@ dependencies:
   - pytest>=7.0.0
   - pytest-cov
   - pytest-xdist>=2.2.0
-  - pytest-asyncio>=0.17
+  - pytest-asyncio>=0.17.0
   - boto3
 
   # required dependencies
@@ -22,37 +22,37 @@ dependencies:
   - pytz
 
   # optional dependencies
-  - beautifulsoup4
+  - beautifulsoup4>=4.9.3
   - blosc
-  - brotlipy
-  - bottleneck
-  - fastparquet
-  - fsspec
-  - html5lib
-  - hypothesis
-  - gcsfs
-  - jinja2
-  - lxml
+  - brotlipy>=0.7.0
+  - bottleneck>=1.3.2
+  - fastparquet>=0.6.3
+  - fsspec>=2021.07.0
+  - html5lib>=1.1
+  - hypothesis>=6.34.2
+  - gcsfs>=2021.07.0
+  - jinja2>=3.0.0
+  - lxml>=4.6.3
   - matplotlib>=3.6.1, <3.7.0
-  - numba
-  - numexpr
-  - openpyxl<3.1.1
-  - odfpy
-  - psycopg2
+  - numba>=0.53.1
+  - numexpr>=2.7.3
+  - openpyxl<3.1.1, >=3.0.7
+  - odfpy>=1.4.1
+  - psycopg2>=2.8.6
   - pyarrow
-  - pymysql
-  - pyreadstat
-  - pytables
-  - python-snappy
-  - pyxlsb
+  - pymysql>=1.0.2
+  - pyreadstat>=1.1.2
+  - pytables>=3.6.1
+  - python-snappy>=0.6.0
+  - pyxlsb>=1.0.8
   - s3fs>=2021.08.0
-  - scipy
-  - sqlalchemy
-  - tabulate
-  - xarray
-  - xlrd
-  - xlsxwriter
-  - zstandard
+  - scipy>=1.7.1
+  - sqlalchemy>=1.4.16
+  - tabulate>=0.8.9
+  - xarray>=0.21.0
+  - xlrd>=2.0.1
+  - xlsxwriter>=1.4.3
+  - zstandard>=0.15.2
 
   # downstream packages
   - botocore
@@ -65,6 +65,6 @@ dependencies:
   - statsmodels
   - coverage
   - pandas-datareader
-  - pandas-gbq
+  - pandas-gbq>=0.15.0
   - pyyaml
   - py

ci/deps/actions-38-minimum_versions.yaml

@@ -14,7 +14,7 @@ dependencies:
   - pytest>=7.0.0
   - pytest-cov
   - pytest-xdist>=2.2.0
-  - pytest-asyncio>=0.17
+  - pytest-asyncio>=0.17.0
   - boto3
 
   # required dependencies

ci/deps/actions-38.yaml

@@ -12,7 +12,7 @@ dependencies:
   - pytest>=7.0.0
   - pytest-cov
   - pytest-xdist>=2.2.0
-  - pytest-asyncio>=0.17
+  - pytest-asyncio>=0.17.0
   - boto3
 
   # required dependencies
@@ -21,35 +21,35 @@ dependencies:
   - pytz
 
   # optional dependencies
-  - beautifulsoup4
+  - beautifulsoup4>=4.9.3
   - blosc
-  - bottleneck
-  - brotlipy
-  - fastparquet
-  - fsspec
-  - html5lib
-  - hypothesis
-  - gcsfs
-  - jinja2
-  - lxml
+  - bottleneck>=1.3.2
+  - brotlipy>=0.7.0
+  - fastparquet>=0.6.3
+  - fsspec>=2021.07.0
+  - html5lib>=1.1
+  - hypothesis>=6.34.2
+  - gcsfs>=2021.07.0
+  - jinja2>=3.0.0
+  - lxml>=4.6.3
   - matplotlib>=3.6.1, <3.7.0
-  - numba
-  - numexpr
-  - openpyxl<3.1.1
-  - odfpy
-  - pandas-gbq
-  - psycopg2
+  - numba>=0.53.1
+  - numexpr>=2.7.3
+  - openpyxl<3.1.1, >=3.0.7
+  - odfpy>=1.4.1
+  - pandas-gbq>=0.15.0
+  - psycopg2>=2.8.6
   - pyarrow
-  - pymysql
-  - pyreadstat
-  - pytables
-  - python-snappy
-  - pyxlsb
+  - pymysql>=1.0.2
+  - pyreadstat>=1.1.2
+  - pytables>=3.6.1
+  - python-snappy>=0.6.0
+  - pyxlsb>=1.0.8
   - s3fs>=2021.08.0
-  - scipy
-  - sqlalchemy
-  - tabulate
-  - xarray
-  - xlrd
-  - xlsxwriter
-  - zstandard
+  - scipy>=1.7.1
+  - sqlalchemy>=1.4.16
+  - tabulate>=0.8.9
+  - xarray>=0.21.0
+  - xlrd>=2.0.1
+  - xlsxwriter>=1.4.3
+  - zstandard>=0.15.2