From fedc0016e385800ceb2833fbcdfb1e98db584e73 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Wed, 29 Dec 2021 16:31:15 -0800 Subject: [PATCH 01/12] CI: Migrate Python 3.10 testing to Posix GHA/Azure Pipelines --- .github/workflows/python-dev.yml | 190 +++++++++--------- ...umpydev.yaml => actions-310-numpydev.yaml} | 2 +- ci/deps/actions-310-slow.yaml | 41 ++++ ci/deps/actions-310.yaml | 40 ++++ ci/deps/azure-macos-310.yaml | 36 ++++ ci/deps/azure-macos-38.yaml | 4 +- ci/deps/azure-macos-39.yaml | 36 ++++ ci/deps/azure-windows-310.yaml | 40 ++++ 8 files changed, 294 insertions(+), 95 deletions(-) rename ci/deps/{actions-39-numpydev.yaml => actions-310-numpydev.yaml} (95%) create mode 100644 ci/deps/actions-310-slow.yaml create mode 100644 ci/deps/actions-310.yaml create mode 100644 ci/deps/azure-macos-310.yaml create mode 100644 ci/deps/azure-macos-39.yaml create mode 100644 ci/deps/azure-windows-310.yaml diff --git a/.github/workflows/python-dev.yml b/.github/workflows/python-dev.yml index a7c7f3b739f65..d23be2755fd86 100644 --- a/.github/workflows/python-dev.yml +++ b/.github/workflows/python-dev.yml @@ -1,92 +1,100 @@ -name: Python Dev +# This file is purposely frozen. DO NOT DELETE IT +# Unfreeze once the next Python Dev version has released beta 1 +# and both Cython and numpy support it +# After that Python has released, migrate the workflows to the +# posix GHA workflows/Azure pipelines and "freeze" this file by +# commenting it out. +# Feel free to modify this comment as necessary. -on: - push: - branches: - - master - - 1.3.x - pull_request: - branches: - - master - - 1.3.x - paths-ignore: - - "doc/**" - -env: - PYTEST_WORKERS: "auto" - PANDAS_CI: 1 - PATTERN: "not slow and not network and not clipboard" - COVERAGE: true - PYTEST_TARGET: pandas - -jobs: - build: - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macOS-latest, windows-latest] - - name: actions-310-dev - timeout-minutes: 80 - - concurrency: - #https://github.community/t/concurrecy-not-work-for-push/183068/7 - group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.os }}-${{ matrix.pytest_target }}-dev - cancel-in-progress: true - - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Set up Python Dev Version - uses: actions/setup-python@v2 - with: - python-version: '3.10-dev' - - # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 - - name: Install dependencies - shell: bash - run: | - python -m pip install --upgrade pip "setuptools<60.0.0" wheel - pip install -i https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy - pip install git+https://github.com/nedbat/coveragepy.git - pip install cython python-dateutil pytz hypothesis pytest>=6.2.5 pytest-xdist pytest-cov - pip list - - - name: Build Pandas - run: | - python setup.py build_ext -q -j2 - python -m pip install -e . --no-build-isolation --no-use-pep517 - - - name: Build Version - run: | - python -c "import pandas; pandas.show_versions();" - - - name: Test with pytest - shell: bash - run: | - ci/run_tests.sh - - - name: Publish test results - uses: actions/upload-artifact@master - with: - name: Test results - path: test-data.xml - if: failure() - - - name: Print skipped tests - run: | - python ci/print_skipped.py - - - name: Report Coverage - run: | - coverage report -m - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v2 - with: - flags: unittests - name: codecov-pandas - fail_ci_if_error: true +#name: Python Dev +# +#on: +# push: +# branches: +# - master +# - 1.3.x +# pull_request: +# branches: +# - master +# - 1.3.x +# paths-ignore: +# - "doc/**" +# +#env: +# PYTEST_WORKERS: "auto" +# PANDAS_CI: 1 +# PATTERN: "not slow and not network and not clipboard" +# COVERAGE: true +# PYTEST_TARGET: pandas +# +#jobs: +# build: +# runs-on: ${{ matrix.os }} +# strategy: +# fail-fast: false +# matrix: +# os: [ubuntu-latest, macOS-latest, windows-latest] +# +# name: actions-310-dev +# timeout-minutes: 80 +# +# concurrency: +# #https://github.community/t/concurrecy-not-work-for-push/183068/7 +# group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.os }}-${{ matrix.pytest_target }}-dev +# cancel-in-progress: true +# +# steps: +# - uses: actions/checkout@v2 +# with: +# fetch-depth: 0 +# +# - name: Set up Python Dev Version +# uses: actions/setup-python@v2 +# with: +# python-version: '3.11-dev' +# +# # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 +# - name: Install dependencies +# shell: bash +# run: | +# python -m pip install --upgrade pip "setuptools<60.0.0" wheel +# pip install -i https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy +# pip install git+https://github.com/nedbat/coveragepy.git +# pip install cython python-dateutil pytz hypothesis pytest>=6.2.5 pytest-xdist pytest-cov +# pip list +# +# - name: Build Pandas +# run: | +# python setup.py build_ext -q -j2 +# python -m pip install -e . --no-build-isolation --no-use-pep517 +# +# - name: Build Version +# run: | +# python -c "import pandas; pandas.show_versions();" +# +# - name: Test with pytest +# shell: bash +# run: | +# ci/run_tests.sh +# +# - name: Publish test results +# uses: actions/upload-artifact@master +# with: +# name: Test results +# path: test-data.xml +# if: failure() +# +# - name: Print skipped tests +# run: | +# python ci/print_skipped.py +# +# - name: Report Coverage +# run: | +# coverage report -m +# +# - name: Upload coverage to Codecov +# uses: codecov/codecov-action@v2 +# with: +# flags: unittests +# name: codecov-pandas +# fail_ci_if_error: true diff --git a/ci/deps/actions-39-numpydev.yaml b/ci/deps/actions-310-numpydev.yaml similarity index 95% rename from ci/deps/actions-39-numpydev.yaml rename to ci/deps/actions-310-numpydev.yaml index 4a6acf55e265f..3e32665d5433f 100644 --- a/ci/deps/actions-39-numpydev.yaml +++ b/ci/deps/actions-310-numpydev.yaml @@ -2,7 +2,7 @@ name: pandas-dev channels: - defaults dependencies: - - python=3.9 + - python=3.10 # tools - pytest>=6.0 diff --git a/ci/deps/actions-310-slow.yaml b/ci/deps/actions-310-slow.yaml new file mode 100644 index 0000000000000..8ea284d24ed3d --- /dev/null +++ b/ci/deps/actions-310-slow.yaml @@ -0,0 +1,41 @@ +name: pandas-dev +channels: + - defaults + - conda-forge +dependencies: + - python=3.10 + + # tools + - cython>=0.29.24 + - pytest>=6.0 + - pytest-cov + - pytest-xdist>=1.31 + - hypothesis>=5.5.3 + + # pandas dependencies + - beautifulsoup4 + - bottleneck + - boto3 + - fsspec>=0.8.0 + - gcsfs + - html5lib + - jinja2 + - lxml + - matplotlib + - numba + - numexpr + - numpy + - openpyxl + - pyarrow>2.0.1 + - pytables + - python-dateutil + - pytz + - s3fs>=0.4.2 + - scipy + - sqlalchemy + - xlrd + - xlsxwriter + - xlwt + - pyreadstat + - pyxlsb + - zstandard diff --git a/ci/deps/actions-310.yaml b/ci/deps/actions-310.yaml new file mode 100644 index 0000000000000..6bf5e1ddbebc6 --- /dev/null +++ b/ci/deps/actions-310.yaml @@ -0,0 +1,40 @@ +name: pandas-dev +channels: + - conda-forge +dependencies: + - python=3.10 + + # tools + - cython>=0.29.24 + - pytest>=6.0 + - pytest-cov + - pytest-xdist>=1.31 + - hypothesis>=5.5.3 + + # pandas dependencies + - beautifulsoup4 + - bottleneck + - boto3 + - fsspec>=0.8.0 + - gcsfs + - html5lib + - jinja2 + - lxml + - matplotlib + - numpy # move up to solve before numba + - numba + - numexpr + - openpyxl + - pyarrow>2.0.1 + - pytables + - python-dateutil + - pytz + - s3fs>=0.4.2 + - scipy + - sqlalchemy + - xlrd + - xlsxwriter + - xlwt + - pyreadstat + - pyxlsb + - zstandard diff --git a/ci/deps/azure-macos-310.yaml b/ci/deps/azure-macos-310.yaml new file mode 100644 index 0000000000000..eb984df1aa739 --- /dev/null +++ b/ci/deps/azure-macos-310.yaml @@ -0,0 +1,36 @@ +name: pandas-dev +channels: + - defaults + - conda-forge +dependencies: + - python=3.10 + + # tools + - cython>=0.29.24 + - pytest>=6.0 + - pytest-xdist>=1.31 + - hypothesis>=5.5.3 + - pytest-azurepipelines + + # pandas dependencies + - beautifulsoup4 + - bottleneck + - html5lib + - jinja2 + - lxml + - matplotlib=3.3.2 + - nomkl + - numexpr + - numpy=1.21.3 + - openpyxl + - pyarrow=1.0.1 + - pyreadstat + - pytables + - python-dateutil==2.8.1 + - pytz + - pyxlsb + - xarray + - xlrd + - xlsxwriter + - xlwt + - zstandard diff --git a/ci/deps/azure-macos-38.yaml b/ci/deps/azure-macos-38.yaml index 472dc8754d13e..422aa86c57fc7 100644 --- a/ci/deps/azure-macos-38.yaml +++ b/ci/deps/azure-macos-38.yaml @@ -6,6 +6,7 @@ dependencies: - python=3.8 # tools + - cython>=0.29.24 - pytest>=6.0 - pytest-xdist>=1.31 - hypothesis>=5.5.3 @@ -33,6 +34,3 @@ dependencies: - xlsxwriter - xlwt - zstandard - - pip - - pip: - - cython>=0.29.24 diff --git a/ci/deps/azure-macos-39.yaml b/ci/deps/azure-macos-39.yaml new file mode 100644 index 0000000000000..a0860ef536953 --- /dev/null +++ b/ci/deps/azure-macos-39.yaml @@ -0,0 +1,36 @@ +name: pandas-dev +channels: + - defaults + - conda-forge +dependencies: + - python=3.9 + + # tools + - cython>=0.29.24 + - pytest>=6.0 + - pytest-xdist>=1.31 + - hypothesis>=5.5.3 + - pytest-azurepipelines + + # pandas dependencies + - beautifulsoup4 + - bottleneck + - html5lib + - jinja2 + - lxml + - matplotlib=3.3.2 + - nomkl + - numexpr + - numpy=1.21.3 + - openpyxl + - pyarrow=1.0.1 + - pyreadstat + - pytables + - python-dateutil==2.8.1 + - pytz + - pyxlsb + - xarray + - xlrd + - xlsxwriter + - xlwt + - zstandard diff --git a/ci/deps/azure-windows-310.yaml b/ci/deps/azure-windows-310.yaml new file mode 100644 index 0000000000000..ccbae9afcdd38 --- /dev/null +++ b/ci/deps/azure-windows-310.yaml @@ -0,0 +1,40 @@ +name: pandas-dev +channels: + - conda-forge + - defaults +dependencies: + - python=3.10 + + # tools + - cython>=0.29.24 + - pytest>=6.0 + - pytest-xdist>=1.31 + - hypothesis>=5.5.3 + - pytest-azurepipelines + + # pandas dependencies + - beautifulsoup4 + - bottleneck + - fsspec>=0.8.0 + - gcsfs + - html5lib + - jinja2 + - lxml + - matplotlib + - numba + - numexpr + - numpy + - openpyxl + - pyarrow + - pytables + - python-dateutil + - pytz + - s3fs>=0.4.2 + - scipy + - sqlalchemy + - xlrd + - xlsxwriter + - xlwt + - pyreadstat + - pyxlsb + - zstandard From 72bc95ba698975f5620b50fa2577c9aa131aeb00 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Wed, 29 Dec 2021 18:53:09 -0800 Subject: [PATCH 02/12] Update posix.yml --- .github/workflows/posix.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/posix.yml b/.github/workflows/posix.yml index 2a40be680c681..8837129024ec6 100644 --- a/.github/workflows/posix.yml +++ b/.github/workflows/posix.yml @@ -34,8 +34,10 @@ jobs: [actions-38-locale.yaml, "not slow and not network", "language-pack-zh-hans xsel", "zh_CN.utf8", "zh_CN.utf8", "", ""], [actions-39-slow.yaml, "slow", "", "", "", "", ""], [actions-pypy-38.yaml, "not slow and not clipboard", "", "", "", "", "--max-worker-restart 0"], - [actions-39-numpydev.yaml, "not slow and not network", "xsel", "", "", "deprecate", "-W error"], - [actions-39.yaml, "not slow and not clipboard", "", "", "", "", ""] + [actions-310-numpydev.yaml, "not slow and not network", "xsel", "", "", "deprecate", "-W error"], + [actions-39.yaml, "not slow and not clipboard", "", "", "", "", ""], + [actions-310.yaml, "not slow and not clipboard", "", "", "", "", ""], + [actions-310-slow.yaml, "slow", "", "", "", "", ""], ] fail-fast: false env: From 6a6cfe0bd48b20e26ba3fbee293af29d55a2933b Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Wed, 29 Dec 2021 18:55:07 -0800 Subject: [PATCH 03/12] Update posix.yml --- ci/azure/posix.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index b7c36bb87353b..02a4a9ad44865 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -18,6 +18,26 @@ jobs: CONDA_PY: "38" PATTERN: "not slow" PYTEST_TARGET: "pandas/tests/[i-z]*" + py39_macos_1: + ENV_FILE: ci/deps/azure-macos-39.yaml + CONDA_PY: "39" + PATTERN: "not slow" + PYTEST_TARGET: "pandas/tests/[a-h]*" + py39_macos_2: + ENV_FILE: ci/deps/azure-macos-39.yaml + CONDA_PY: "39" + PATTERN: "not slow" + PYTEST_TARGET: "pandas/tests/[i-z]*" + py310_macos_1: + ENV_FILE: ci/deps/azure-macos-310.yaml + CONDA_PY: "310" + PATTERN: "not slow" + PYTEST_TARGET: "pandas/tests/[a-h]*" + py310_macos_2: + ENV_FILE: ci/deps/azure-macos-310.yaml + CONDA_PY: "310" + PATTERN: "not slow" + PYTEST_TARGET: "pandas/tests/[i-z]*" steps: - script: echo '##vso[task.prependpath]$(HOME)/miniconda3/bin' From 3b1eb1c07f11ea6b885e8da39cefe792e345d5a4 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Wed, 29 Dec 2021 18:56:06 -0800 Subject: [PATCH 04/12] Update windows.yml --- ci/azure/windows.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/ci/azure/windows.yml b/ci/azure/windows.yml index 7f3efb5a4dbf3..4c6509390ab45 100644 --- a/ci/azure/windows.yml +++ b/ci/azure/windows.yml @@ -35,6 +35,20 @@ jobs: PATTERN: "not slow and not high_memory" PYTEST_WORKERS: 2 # GH-42236 PYTEST_TARGET: "pandas/tests/[i-z]*" + + py310_1: + ENV_FILE: ci/deps/azure-windows-310.yaml + CONDA_PY: "310" + PATTERN: "not slow and not high_memory" + PYTEST_WORKERS: 2 # GH-42236 + PYTEST_TARGET: "pandas/tests/[a-h]*" + + py310_2: + ENV_FILE: ci/deps/azure-windows-310.yaml + CONDA_PY: "310" + PATTERN: "not slow and not high_memory" + PYTEST_WORKERS: 2 # GH-42236 + PYTEST_TARGET: "pandas/tests/[i-z]*" steps: - powershell: | From 14049dbbb8e48c7b67091745ce94711e9834ed6b Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Wed, 29 Dec 2021 18:59:45 -0800 Subject: [PATCH 05/12] Update python-dev.yml --- .github/workflows/python-dev.yml | 193 ++++++++++++++++--------------- 1 file changed, 97 insertions(+), 96 deletions(-) diff --git a/.github/workflows/python-dev.yml b/.github/workflows/python-dev.yml index d23be2755fd86..d9d50de27cce7 100644 --- a/.github/workflows/python-dev.yml +++ b/.github/workflows/python-dev.yml @@ -1,100 +1,101 @@ -# This file is purposely frozen. DO NOT DELETE IT -# Unfreeze once the next Python Dev version has released beta 1 -# and both Cython and numpy support it +# This file is purposely frozen(does not run). DO NOT DELETE IT +# Unfreeze(by commentingthe if: false() condition) once the +# next Python Dev version has released beta 1 and both Cython and numpy support it # After that Python has released, migrate the workflows to the # posix GHA workflows/Azure pipelines and "freeze" this file by -# commenting it out. +# uncommenting the if: false() condition # Feel free to modify this comment as necessary. -#name: Python Dev -# -#on: -# push: -# branches: -# - master -# - 1.3.x -# pull_request: -# branches: -# - master -# - 1.3.x -# paths-ignore: -# - "doc/**" -# -#env: -# PYTEST_WORKERS: "auto" -# PANDAS_CI: 1 -# PATTERN: "not slow and not network and not clipboard" -# COVERAGE: true -# PYTEST_TARGET: pandas -# -#jobs: -# build: -# runs-on: ${{ matrix.os }} -# strategy: -# fail-fast: false -# matrix: -# os: [ubuntu-latest, macOS-latest, windows-latest] -# -# name: actions-310-dev -# timeout-minutes: 80 -# -# concurrency: -# #https://github.community/t/concurrecy-not-work-for-push/183068/7 -# group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.os }}-${{ matrix.pytest_target }}-dev -# cancel-in-progress: true -# -# steps: -# - uses: actions/checkout@v2 -# with: -# fetch-depth: 0 -# -# - name: Set up Python Dev Version -# uses: actions/setup-python@v2 -# with: -# python-version: '3.11-dev' -# -# # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 -# - name: Install dependencies -# shell: bash -# run: | -# python -m pip install --upgrade pip "setuptools<60.0.0" wheel -# pip install -i https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy -# pip install git+https://github.com/nedbat/coveragepy.git -# pip install cython python-dateutil pytz hypothesis pytest>=6.2.5 pytest-xdist pytest-cov -# pip list -# -# - name: Build Pandas -# run: | -# python setup.py build_ext -q -j2 -# python -m pip install -e . --no-build-isolation --no-use-pep517 -# -# - name: Build Version -# run: | -# python -c "import pandas; pandas.show_versions();" -# -# - name: Test with pytest -# shell: bash -# run: | -# ci/run_tests.sh -# -# - name: Publish test results -# uses: actions/upload-artifact@master -# with: -# name: Test results -# path: test-data.xml -# if: failure() -# -# - name: Print skipped tests -# run: | -# python ci/print_skipped.py -# -# - name: Report Coverage -# run: | -# coverage report -m -# -# - name: Upload coverage to Codecov -# uses: codecov/codecov-action@v2 -# with: -# flags: unittests -# name: codecov-pandas -# fail_ci_if_error: true +name: Python Dev + +on: + push: + branches: + - master + - 1.3.x + pull_request: + branches: + - master + - 1.3.x + paths-ignore: + - "doc/**" + +env: + PYTEST_WORKERS: "auto" + PANDAS_CI: 1 + PATTERN: "not slow and not network and not clipboard" + COVERAGE: true + PYTEST_TARGET: pandas + +jobs: + build: + if: false() # Comment this line out to "unfreeze" + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macOS-latest, windows-latest] + + name: actions-310-dev + timeout-minutes: 80 + + concurrency: + #https://github.community/t/concurrecy-not-work-for-push/183068/7 + group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.os }}-${{ matrix.pytest_target }}-dev + cancel-in-progress: true + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Set up Python Dev Version + uses: actions/setup-python@v2 + with: + python-version: '3.11-dev' + + # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 + - name: Install dependencies + shell: bash + run: | + python -m pip install --upgrade pip "setuptools<60.0.0" wheel + pip install -i https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy + pip install git+https://github.com/nedbat/coveragepy.git + pip install cython python-dateutil pytz hypothesis pytest>=6.2.5 pytest-xdist pytest-cov + pip list + + - name: Build Pandas + run: | + python setup.py build_ext -q -j2 + python -m pip install -e . --no-build-isolation --no-use-pep517 + + - name: Build Version + run: | + python -c "import pandas; pandas.show_versions();" + + - name: Test with pytest + shell: bash + run: | + ci/run_tests.sh + + - name: Publish test results + uses: actions/upload-artifact@master + with: + name: Test results + path: test-data.xml + if: failure() + + - name: Print skipped tests + run: | + python ci/print_skipped.py + + - name: Report Coverage + run: | + coverage report -m + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v2 + with: + flags: unittests + name: codecov-pandas + fail_ci_if_error: true From 4fac892e0ace7e9c1855854c5fc817632627e17e Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Wed, 29 Dec 2021 19:29:34 -0800 Subject: [PATCH 06/12] spacing --- .github/workflows/python-dev.yml | 172 +++++++++++++++---------------- 1 file changed, 86 insertions(+), 86 deletions(-) diff --git a/.github/workflows/python-dev.yml b/.github/workflows/python-dev.yml index d9d50de27cce7..c8af791ef32f4 100644 --- a/.github/workflows/python-dev.yml +++ b/.github/workflows/python-dev.yml @@ -9,93 +9,93 @@ name: Python Dev on: - push: - branches: - - master - - 1.3.x - pull_request: - branches: - - master - - 1.3.x - paths-ignore: - - "doc/**" + push: + branches: + - master + - 1.3.x + pull_request: + branches: + - master + - 1.3.x + paths-ignore: + - "doc/**" env: - PYTEST_WORKERS: "auto" - PANDAS_CI: 1 - PATTERN: "not slow and not network and not clipboard" - COVERAGE: true - PYTEST_TARGET: pandas + PYTEST_WORKERS: "auto" + PANDAS_CI: 1 + PATTERN: "not slow and not network and not clipboard" + COVERAGE: true + PYTEST_TARGET: pandas jobs: - build: - if: false() # Comment this line out to "unfreeze" - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macOS-latest, windows-latest] - - name: actions-310-dev - timeout-minutes: 80 - - concurrency: - #https://github.community/t/concurrecy-not-work-for-push/183068/7 - group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.os }}-${{ matrix.pytest_target }}-dev - cancel-in-progress: true - - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Set up Python Dev Version - uses: actions/setup-python@v2 - with: - python-version: '3.11-dev' - - # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 - - name: Install dependencies - shell: bash - run: | - python -m pip install --upgrade pip "setuptools<60.0.0" wheel - pip install -i https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy - pip install git+https://github.com/nedbat/coveragepy.git - pip install cython python-dateutil pytz hypothesis pytest>=6.2.5 pytest-xdist pytest-cov - pip list - - - name: Build Pandas - run: | - python setup.py build_ext -q -j2 - python -m pip install -e . --no-build-isolation --no-use-pep517 - - - name: Build Version - run: | - python -c "import pandas; pandas.show_versions();" - - - name: Test with pytest - shell: bash - run: | - ci/run_tests.sh - - - name: Publish test results - uses: actions/upload-artifact@master - with: - name: Test results - path: test-data.xml - if: failure() - - - name: Print skipped tests - run: | - python ci/print_skipped.py - - - name: Report Coverage - run: | - coverage report -m - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v2 - with: - flags: unittests - name: codecov-pandas - fail_ci_if_error: true + build: + if: false # Comment this line out to "unfreeze" + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macOS-latest, windows-latest] + + name: actions-310-dev + timeout-minutes: 80 + + concurrency: + #https://github.community/t/concurrecy-not-work-for-push/183068/7 + group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.os }}-${{ matrix.pytest_target }}-dev + cancel-in-progress: true + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Set up Python Dev Version + uses: actions/setup-python@v2 + with: + python-version: '3.11-dev' + + # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 + - name: Install dependencies + shell: bash + run: | + python -m pip install --upgrade pip "setuptools<60.0.0" wheel + pip install -i https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy + pip install git+https://github.com/nedbat/coveragepy.git + pip install cython python-dateutil pytz hypothesis pytest>=6.2.5 pytest-xdist pytest-cov + pip list + + - name: Build Pandas + run: | + python setup.py build_ext -q -j2 + python -m pip install -e . --no-build-isolation --no-use-pep517 + + - name: Build Version + run: | + python -c "import pandas; pandas.show_versions();" + + - name: Test with pytest + shell: bash + run: | + ci/run_tests.sh + + - name: Publish test results + uses: actions/upload-artifact@master + with: + name: Test results + path: test-data.xml + if: failure() + + - name: Print skipped tests + run: | + python ci/print_skipped.py + + - name: Report Coverage + run: | + coverage report -m + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v2 + with: + flags: unittests + name: codecov-pandas + fail_ci_if_error: true From 98bcb764b8992d383531c1519ad62fe5f6c1a0c5 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Tue, 4 Jan 2022 20:48:16 -0800 Subject: [PATCH 07/12] remove numba --- ci/deps/actions-310-slow.yaml | 3 ++- ci/deps/actions-310.yaml | 3 ++- ci/deps/azure-macos-310.yaml | 2 +- ci/deps/azure-windows-310.yaml | 3 ++- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/ci/deps/actions-310-slow.yaml b/ci/deps/actions-310-slow.yaml index 8ea284d24ed3d..32bf35670b13b 100644 --- a/ci/deps/actions-310-slow.yaml +++ b/ci/deps/actions-310-slow.yaml @@ -22,7 +22,8 @@ dependencies: - jinja2 - lxml - matplotlib - - numba + # TODO: uncomment after numba supports py310 + #- numba - numexpr - numpy - openpyxl diff --git a/ci/deps/actions-310.yaml b/ci/deps/actions-310.yaml index 6bf5e1ddbebc6..6516a76a393d5 100644 --- a/ci/deps/actions-310.yaml +++ b/ci/deps/actions-310.yaml @@ -22,7 +22,8 @@ dependencies: - lxml - matplotlib - numpy # move up to solve before numba - - numba + # TODO: uncomment after numba supports py310 + #- numba - numexpr - openpyxl - pyarrow>2.0.1 diff --git a/ci/deps/azure-macos-310.yaml b/ci/deps/azure-macos-310.yaml index eb984df1aa739..8993e2249982f 100644 --- a/ci/deps/azure-macos-310.yaml +++ b/ci/deps/azure-macos-310.yaml @@ -18,7 +18,7 @@ dependencies: - html5lib - jinja2 - lxml - - matplotlib=3.3.2 + - matplotlib - nomkl - numexpr - numpy=1.21.3 diff --git a/ci/deps/azure-windows-310.yaml b/ci/deps/azure-windows-310.yaml index ccbae9afcdd38..8e6f4deef6057 100644 --- a/ci/deps/azure-windows-310.yaml +++ b/ci/deps/azure-windows-310.yaml @@ -21,7 +21,8 @@ dependencies: - jinja2 - lxml - matplotlib - - numba + # TODO: uncomment after numba supports py310 + #- numba - numexpr - numpy - openpyxl From 859908de5333cad3efa02ef93760932ee5b3c593 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Sat, 8 Jan 2022 09:37:43 -0800 Subject: [PATCH 08/12] fixes --- .github/workflows/python-dev.yml | 12 ++++++------ ci/azure/windows.yml | 2 +- ci/deps/azure-macos-310.yaml | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/python-dev.yml b/.github/workflows/python-dev.yml index c8af791ef32f4..f96127ef0ab8f 100644 --- a/.github/workflows/python-dev.yml +++ b/.github/workflows/python-dev.yml @@ -1,5 +1,5 @@ # This file is purposely frozen(does not run). DO NOT DELETE IT -# Unfreeze(by commentingthe if: false() condition) once the +# Unfreeze(by commentingthe if: false() condition) once the # next Python Dev version has released beta 1 and both Cython and numpy support it # After that Python has released, migrate the workflows to the # posix GHA workflows/Azure pipelines and "freeze" this file by @@ -21,11 +21,11 @@ on: - "doc/**" env: - PYTEST_WORKERS: "auto" - PANDAS_CI: 1 - PATTERN: "not slow and not network and not clipboard" - COVERAGE: true - PYTEST_TARGET: pandas + PYTEST_WORKERS: "auto" + PANDAS_CI: 1 + PATTERN: "not slow and not network and not clipboard" + COVERAGE: true + PYTEST_TARGET: pandas jobs: build: diff --git a/ci/azure/windows.yml b/ci/azure/windows.yml index 4c6509390ab45..7061a266f28c7 100644 --- a/ci/azure/windows.yml +++ b/ci/azure/windows.yml @@ -35,7 +35,7 @@ jobs: PATTERN: "not slow and not high_memory" PYTEST_WORKERS: 2 # GH-42236 PYTEST_TARGET: "pandas/tests/[i-z]*" - + py310_1: ENV_FILE: ci/deps/azure-windows-310.yaml CONDA_PY: "310" diff --git a/ci/deps/azure-macos-310.yaml b/ci/deps/azure-macos-310.yaml index 8993e2249982f..312fac8091db6 100644 --- a/ci/deps/azure-macos-310.yaml +++ b/ci/deps/azure-macos-310.yaml @@ -21,9 +21,9 @@ dependencies: - matplotlib - nomkl - numexpr - - numpy=1.21.3 + - numpy - openpyxl - - pyarrow=1.0.1 + - pyarrow - pyreadstat - pytables - python-dateutil==2.8.1 From 98757e19b2a4230e3ef88894b734f211aa70e52d Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Sun, 9 Jan 2022 07:40:56 -0800 Subject: [PATCH 09/12] adjust regex for py310 --- pandas/tests/plotting/frame/test_frame.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/plotting/frame/test_frame.py b/pandas/tests/plotting/frame/test_frame.py index 6c07366e402d6..ff247349ff4d5 100644 --- a/pandas/tests/plotting/frame/test_frame.py +++ b/pandas/tests/plotting/frame/test_frame.py @@ -682,7 +682,7 @@ def test_raise_error_on_datetime_time_data(self): # GH 8113, datetime.time type is not supported by matplotlib in scatter df = DataFrame(np.random.randn(10), columns=["a"]) df["dtime"] = date_range(start="2014-01-01", freq="h", periods=10).time - msg = "must be a string or a number, not 'datetime.time'" + msg = "must be a string or a (real )?number, not 'datetime.time'" with pytest.raises(TypeError, match=msg): df.plot(kind="scatter", x="dtime", y="a") From 72318899d87bfafac103bffb513240742354bf2f Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Sun, 9 Jan 2022 12:44:23 -0800 Subject: [PATCH 10/12] try xclip instead of xsel? --- .github/workflows/posix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/posix.yml b/.github/workflows/posix.yml index 8eb74f03e2a4d..8e99d81566022 100644 --- a/.github/workflows/posix.yml +++ b/.github/workflows/posix.yml @@ -34,7 +34,7 @@ jobs: [actions-38-locale.yaml, "not slow and not network", "language-pack-zh-hans xsel", "zh_CN.utf8", "zh_CN.utf8", "", ""], [actions-39-slow.yaml, "slow", "", "", "", "", ""], [actions-pypy-38.yaml, "not slow and not clipboard", "", "", "", "", "--max-worker-restart 0"], - [actions-310-numpydev.yaml, "not slow and not network", "xsel", "", "", "deprecate", "-W error"], + [actions-310-numpydev.yaml, "not slow and not network", "xclip", "", "", "deprecate", "-W error"], [actions-39.yaml, "not slow and not clipboard", "", "", "", "", ""], [actions-310.yaml, "not slow and not clipboard", "", "", "", "", ""], [actions-310-slow.yaml, "slow", "", "", "", "", ""], From 248290cb8f35392dd01da2f47267f7735455d852 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Mon, 10 Jan 2022 15:52:02 -0800 Subject: [PATCH 11/12] Update python-dev.yml --- .github/workflows/python-dev.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python-dev.yml b/.github/workflows/python-dev.yml index f96127ef0ab8f..7b78de7c237a5 100644 --- a/.github/workflows/python-dev.yml +++ b/.github/workflows/python-dev.yml @@ -12,11 +12,11 @@ on: push: branches: - master - - 1.3.x + - 1.4.x pull_request: branches: - master - - 1.3.x + - 1.4.x paths-ignore: - "doc/**" @@ -36,7 +36,7 @@ jobs: matrix: os: [ubuntu-latest, macOS-latest, windows-latest] - name: actions-310-dev + name: actions-311-dev timeout-minutes: 80 concurrency: From ff9244f72d3b76d93bd0b98776fe66a6563dfb64 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Sun, 16 Jan 2022 08:22:59 -0800 Subject: [PATCH 12/12] updates --- ci/deps/actions-310-slow.yaml | 42 ----------------------------------- ci/deps/actions-310.yaml | 38 +++++++++++++++++++------------ 2 files changed, 24 insertions(+), 56 deletions(-) delete mode 100644 ci/deps/actions-310-slow.yaml diff --git a/ci/deps/actions-310-slow.yaml b/ci/deps/actions-310-slow.yaml deleted file mode 100644 index 32bf35670b13b..0000000000000 --- a/ci/deps/actions-310-slow.yaml +++ /dev/null @@ -1,42 +0,0 @@ -name: pandas-dev -channels: - - defaults - - conda-forge -dependencies: - - python=3.10 - - # tools - - cython>=0.29.24 - - pytest>=6.0 - - pytest-cov - - pytest-xdist>=1.31 - - hypothesis>=5.5.3 - - # pandas dependencies - - beautifulsoup4 - - bottleneck - - boto3 - - fsspec>=0.8.0 - - gcsfs - - html5lib - - jinja2 - - lxml - - matplotlib - # TODO: uncomment after numba supports py310 - #- numba - - numexpr - - numpy - - openpyxl - - pyarrow>2.0.1 - - pytables - - python-dateutil - - pytz - - s3fs>=0.4.2 - - scipy - - sqlalchemy - - xlrd - - xlsxwriter - - xlwt - - pyreadstat - - pyxlsb - - zstandard diff --git a/ci/deps/actions-310.yaml b/ci/deps/actions-310.yaml index 6516a76a393d5..9829380620f86 100644 --- a/ci/deps/actions-310.yaml +++ b/ci/deps/actions-310.yaml @@ -2,40 +2,50 @@ name: pandas-dev channels: - conda-forge dependencies: - - python=3.10 + - python=3.9 - # tools - - cython>=0.29.24 + # test dependencies + - cython=0.29.24 - pytest>=6.0 - pytest-cov - pytest-xdist>=1.31 - hypothesis>=5.5.3 + - psutil - # pandas dependencies + # required dependencies + - python-dateutil + - numpy + - pytz + + # optional dependencies - beautifulsoup4 + - blosc - bottleneck - - boto3 - - fsspec>=0.8.0 - - gcsfs + - fastparquet + - fsspec - html5lib + - gcsfs - jinja2 - lxml - matplotlib - - numpy # move up to solve before numba # TODO: uncomment after numba supports py310 #- numba - numexpr - openpyxl - - pyarrow>2.0.1 + - odfpy + - pandas-gbq + - psycopg2 + - pymysql - pytables - - python-dateutil - - pytz - - s3fs>=0.4.2 + - pyarrow + - pyreadstat + - pyxlsb + - s3fs - scipy - sqlalchemy + - tabulate + - xarray - xlrd - xlsxwriter - xlwt - - pyreadstat - - pyxlsb - zstandard