DEPS: drop Python 3.7 and NumPy 1.17 from tests (#41678) (#41989)

Author: fangchenli

.github/workflows/database.yml

@@ -19,15 +19,15 @@ env:
   COVERAGE: true
 
 jobs:
-  Linux_py37_IO:
+  Linux_py38_IO:
     runs-on: ubuntu-latest
     defaults:
       run:
         shell: bash -l {0}
 
     strategy:
       matrix:
-        ENV_FILE: [ci/deps/actions-37-db-min.yaml, ci/deps/actions-37-db.yaml]
+        ENV_FILE: [ci/deps/actions-38-db-min.yaml, ci/deps/actions-38-db.yaml]
       fail-fast: false
 
     concurrency:

.github/workflows/posix.yml

@@ -25,14 +25,13 @@ jobs:
     strategy:
       matrix:
         settings: [
-          [actions-37-minimum_versions.yaml, "not slow and not network and not clipboard", "", "", "", "", ""],
-          [actions-37.yaml, "not slow and not network and not clipboard", "", "", "", "", ""],
-          [actions-37-locale_slow.yaml, "slow", "language-pack-it xsel", "it_IT.utf8", "it_IT.utf8", "", ""],
-          [actions-37-slow.yaml, "slow", "", "", "", "", ""],
+          [actions-38-minimum_versions.yaml, "not slow and not network and not clipboard", "", "", "", "", ""],
+          [actions-38-locale_slow.yaml, "slow", "language-pack-it xsel", "it_IT.utf8", "it_IT.utf8", "", ""],
           [actions-38.yaml, "not slow and not network and not clipboard", "", "", "", "", ""],
           [actions-38-slow.yaml, "slow", "", "", "", "", ""],
           [actions-38-locale.yaml, "not slow and not network", "language-pack-zh-hans xsel", "zh_CN.utf8", "zh_CN.utf8", "", ""],
-          [actions-38-numpydev.yaml, "not slow and not network", "xsel", "", "", "deprecate", "-W error"],
+          [actions-39-slow.yaml, "slow", "", "", "", "", ""],
+          [actions-39-numpydev.yaml, "not slow and not network", "xsel", "", "", "deprecate", "-W error"],
           [actions-39.yaml, "not slow and not network and not clipboard", "", "", "", "", ""]
         ]
       fail-fast: false

.github/workflows/sdist.yml

@@ -23,7 +23,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.7", "3.8", "3.9"]
+        python-version: ["3.8", "3.9"]
 
     steps:
     - uses: actions/checkout@v2

azure-pipelines.yml

@@ -29,7 +29,7 @@ jobs:
     name: Windows
     vmImage: vs2017-win2016
 
-- job: py37_32bit
+- job: py38_32bit
   pool:
     vmImage: ubuntu-18.04
 
@@ -38,7 +38,7 @@ jobs:
         docker pull quay.io/pypa/manylinux2014_i686
         docker run -v $(pwd):/pandas quay.io/pypa/manylinux2014_i686 \
         /bin/bash -xc "cd pandas && \
-        /opt/python/cp37-cp37m/bin/python -m venv ~/virtualenvs/pandas-dev && \
+        /opt/python/cp38-cp38/bin/python -m venv ~/virtualenvs/pandas-dev && \
         . ~/virtualenvs/pandas-dev/bin/activate && \
         python -m pip install --no-deps -U pip wheel setuptools && \
         pip install cython numpy python-dateutil pytz pytest pytest-xdist hypothesis pytest-azurepipelines && \
@@ -52,4 +52,4 @@ jobs:
       inputs:
         testResultsFiles: '**/test-*.xml'
         failTaskOnFailedTests: true
-        testRunTitle: 'Publish test results for Python 3.7-32 bit full Linux'
+        testRunTitle: 'Publish test results for Python 3.8-32 bit full Linux'

ci/azure/posix.yml

@@ -9,9 +9,9 @@ jobs:
   strategy:
     matrix:
       ${{ if eq(parameters.name, 'macOS') }}:
-        py37_macos:
-          ENV_FILE: ci/deps/azure-macos-37.yaml
-          CONDA_PY: "37"
+        py38_macos:
+          ENV_FILE: ci/deps/azure-macos-38.yaml
+          CONDA_PY: "38"
           PATTERN: "not slow and not network"
 
   steps:

ci/azure/windows.yml

@@ -8,15 +8,15 @@ jobs:
     vmImage: ${{ parameters.vmImage }}
   strategy:
     matrix:
-      py37_np17:
-        ENV_FILE: ci/deps/azure-windows-37.yaml
-        CONDA_PY: "37"
-        PATTERN: "not slow and not network"
-        PYTEST_WORKERS: 2  # GH-42236
-
       py38_np18:
         ENV_FILE: ci/deps/azure-windows-38.yaml
         CONDA_PY: "38"
+        PATTERN: "not slow and not network"
+        PYTEST_WORKERS: 2  # GH-42236
+
+      py39:
+        ENV_FILE: ci/deps/azure-windows-39.yaml
+        CONDA_PY: "39"
         PATTERN: "not slow and not network and not high_memory"
         PYTEST_WORKERS: 2  # GH-42236
 
@@ -25,26 +25,22 @@ jobs:
         Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
         Write-Host "##vso[task.prependpath]$HOME/miniconda3/bin"
       displayName: 'Add conda to PATH'
-
     - script: conda update -q -n base conda
       displayName: 'Update conda'
 
     - bash: |
         conda env create -q --file ci\\deps\\azure-windows-$(CONDA_PY).yaml
       displayName: 'Create anaconda environment'
-
     - bash: |
         source activate pandas-dev
         conda list
         python setup.py build_ext -q -j 4
         python -m pip install --no-build-isolation -e .
       displayName: 'Build'
-
     - bash: |
         source activate pandas-dev
         ci/run_tests.sh
       displayName: 'Test'
-
     - task: PublishTestResults@2
       condition: succeededOrFailed()
       inputs:

ci/deps/actions-37-locale_slow.yaml

@@ -2,14 +2,14 @@ name: pandas-dev
 channels:
   - conda-forge
 dependencies:
-  - python=3.7.*
+  - python=3.8
 
   # tools
   - cython>=0.29.21
   - pytest>=6.0
   - pytest-cov
-  - pytest-xdist>=1.21
-  - hypothesis>=3.58.0
+  - pytest-xdist>=1.31
+  - hypothesis>=5.5.3
 
   # required
   - numpy<1.20  # GH#39541 compat for pyarrow<3
@@ -18,31 +18,30 @@ dependencies:
 
   # optional
   - beautifulsoup4
-  - blosc=1.17.0
+  - blosc=1.20.1
   - python-blosc
   - fastparquet=0.4.0
   - html5lib
   - ipython
   - jinja2
-  - lxml=4.3.0
+  - lxml=4.5.0
   - matplotlib
   - nomkl
   - numexpr
   - openpyxl
   - pandas-gbq
-  - google-cloud-bigquery>=1.27.2 # GH 36436
   - protobuf>=3.12.4
   - pyarrow=0.17.1 # GH 38803
-  - pytables>=3.5.1
+  - pytables>=3.6.1
   - scipy
-  - xarray=0.12.3
-  - xlrd<2.0
+  - xarray=0.15.1
+  - xlrd
   - xlsxwriter
   - xlwt
   - moto
   - flask
 
   # sql
-  - psycopg2=2.7
-  - pymysql=0.8.1
-  - sqlalchemy=1.3.0
+  - psycopg2=2.8.4
+  - pymysql=0.10.1
+  - sqlalchemy=1.3.11

ci/deps/actions-37-minimum_versions.yaml

@@ -2,13 +2,13 @@ name: pandas-dev
 channels:
   - conda-forge
 dependencies:
-  - python=3.7.*
+  - python=3.8
 
   # tools
   - cython>=0.29.21
   - pytest>=6.0
-  - pytest-xdist>=1.21
-  - hypothesis>=3.58.0
+  - pytest-xdist>=1.31
+  - hypothesis>=5.5.3
   - pytest-cov>=2.10.1  # this is only needed in the coverage build, ref: GH 35737
 
   # pandas dependencies
@@ -25,11 +25,10 @@ dependencies:
   - flask
   - nomkl
   - numexpr
-  - numpy=1.17.*
+  - numpy=1.18
   - odfpy
   - openpyxl
   - pandas-gbq
-  - google-cloud-bigquery>=1.27.2 # GH 36436
   - psycopg2
   - pyarrow>=0.17.0
   - pymysql
@@ -43,7 +42,7 @@ dependencies:
   - sqlalchemy
   - statsmodels
   - xarray
-  - xlrd<2.0
+  - xlrd
   - xlsxwriter
   - xlwt
   - pip

ci/deps/actions-37.yaml

@@ -2,15 +2,15 @@ name: pandas-dev
 channels:
   - conda-forge
 dependencies:
-  - python=3.8.*
+  - python=3.8
 
   # tools
   - cython>=0.29.21
   - pytest>=6.0
   - pytest-cov
-  - pytest-xdist>=1.21
+  - pytest-xdist>=1.31
   - pytest-asyncio>=0.12.0
-  - hypothesis>=3.58.0
+  - hypothesis>=5.5.3
 
   # pandas dependencies
   - beautifulsoup4
@@ -31,7 +31,7 @@ dependencies:
   - pytz
   - scipy
   - xarray
-  - xlrd<2.0
+  - xlrd
   - xlsxwriter
   - xlwt
   - moto

ci/deps/actions-37-db-min.yaml renamed to ci/deps/actions-38-db-min.yaml

@@ -0,0 +1,30 @@
+name: pandas-dev
+channels:
+  - defaults
+  - conda-forge
+dependencies:
+  - python=3.8
+
+  # tools
+  - cython>=0.29.21
+  - pytest>=6.0
+  - pytest-cov
+  - pytest-xdist>=1.31
+  - hypothesis>=5.5.3
+
+  # pandas dependencies
+  - beautifulsoup4=4.8.2
+  - bottleneck=1.3.1
+  - lxml
+  - matplotlib=3.3.2
+  - numpy=1.18
+  - openpyxl=3.0.2
+  - python-dateutil
+  - python-blosc
+  - pytz=2020.1
+  - scipy
+  - sqlalchemy=1.3.11
+  - xlrd=2.0.1
+  - xlsxwriter=1.2.2
+  - xlwt=1.3.0
+  - html5lib=1.1