Skip to content

Commit f5d7550

Browse files
jonashaagjonashaag
committed
Refactor CI
- Drop Azure (move to GHA). - Use mamba-org/provision-with-micromamba to set up Conda envs. - Cache Conda envs. - Use hendrikmuhs/ccache-action to cache C compiliation. - Add Windows 32 bit build. - Drop Ubuntu 32 bit build. - Factor out common GHA code into reusable actions. Undo Ccache and Micromamba changes Remove dead code Fix Undo more changes skip tests Fix skip azure Fix Fix Fix Fix
1 parent 3cb70a5 commit f5d7550

File tree

9 files changed

+266
-82
lines changed

9 files changed

+266
-82
lines changed

‎.github/actions/build-pandas/action.yml

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
name: Build pandas
2+
description: Rebuilds the C extensions and installs pandas
3+
runs:
4+
using: composite
5+
steps:
6+
- name: Environment Detail
7+
run: |
8+
if which mamba 2>/dev/null; then
9+
mamba info
10+
mamba list
11+
mamba info | grep -Ei 'environment.+:' | grep -qEiv 'environment.+:.+none'
12+
fi
13+
pip list
14+
python --version
15+
shell: bash -el {0}
16+
17+
- name: Build Pandas
18+
run: |
19+
which python
20+
which pip
21+
time python setup.py build_ext -v -j 2
22+
pip install -v -e . --no-build-isolation --no-use-pep517 --no-index
23+
shell: bash -el {0}
24+
25+
- name: Build Version
26+
run: pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd
27+
shell: bash -el {0}

‎.github/actions/run-tests/action.yml

+34
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
name: Run tests and report results
2+
inputs:
3+
check-pyarrow-version:
4+
required: false
5+
runs:
6+
using: composite
7+
steps:
8+
- name: Check PyArrow version
9+
run: |
10+
# Double check that we have the expected PyArrow
11+
pushd /tmp
12+
python -c "import pandas; pandas.show_versions()" | egrep -i "pyarrow.+: ${{ matrix.check-pyarrow-version }}"
13+
popd
14+
shell: bash -el {0}
15+
if: ${{ inputs.check-pyarrow-version }}
16+
17+
- name: Test
18+
run: ci/run_tests.sh
19+
shell: bash -el {0}
20+
21+
- name: Publish test results
22+
uses: actions/upload-artifact@v2
23+
with:
24+
name: Test results
25+
path: test-data.xml
26+
if: failure()
27+
28+
- name: Upload coverage to Codecov
29+
uses: codecov/codecov-action@v2
30+
with:
31+
flags: unittests
32+
name: codecov-pandas
33+
fail_ci_if_error: false
34+
if: failure()

‎.github/actions/setup-python/action.yml

+65
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
name: Setup Python and install requirements
2+
inputs:
3+
python-version:
4+
required: true
5+
architecture:
6+
default: x64
7+
runs:
8+
using: composite
9+
steps:
10+
# TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941
11+
- name: Create temporary requirements.txt
12+
run: |
13+
# Drop cache at least once per month
14+
month_today="$(date '+%Y-%m')"
15+
cat > requirements.txt <<EOF
16+
# $month_today
17+
18+
# Python deps
19+
pip
20+
setuptools<60.0.0
21+
wheel
22+
23+
# Pandas deps
24+
# GH 39416
25+
numpy
26+
cython
27+
python-dateutil
28+
pytz
29+
30+
# Test deps
31+
git+https://github.com/nedbat/coveragepy.git
32+
hypothesis
33+
pytest>=6.2.5
34+
pytest-xdist
35+
pytest-cov
36+
pytest-asyncio>=0.17
37+
EOF
38+
shell: bash -el {0}
39+
40+
- name: Set up Python
41+
uses: actions/setup-python@v3
42+
with:
43+
python-version: ${{ inputs.python-version }}
44+
architecture: ${{ inputs.architecture }}
45+
cache: pip
46+
47+
- name: Fix $PATH on macOS
48+
run: |
49+
# On macOS, the Python version we installed above is too late in $PATH
50+
# to be effective if using "bash -l" (which we need for code that works
51+
# with Conda envs).
52+
cat >> ~/.bash_profile <<EOF
53+
export PATH="\$(echo "\$PATH" | grep -Eio '[^:]+hostedtoolcache/python[^:]+bin'):\$PATH" \
54+
EOF
55+
shell: bash -el {0}
56+
if: ${{ runner.os == 'macOS' }}
57+
58+
- name: Install dependencies
59+
run: |
60+
cat requirements.txt
61+
# TODO https://github.com/numpy/numpy/issues/21196
62+
bits32=$(python -c 'import sys; print(int(sys.maxsize > 2**32))')
63+
NPY_DISABLE_SVML=$bits32 pip install -r requirements.txt
64+
pip list
65+
shell: bash -el {0}

‎.github/actions/setup/action.yml

+50-4
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,58 @@
11
name: Set up pandas
22
description: Runs all the setup steps required to have a built pandas ready to use
3+
inputs:
4+
environment-file:
5+
default: environment.yml
6+
pyarrow-version:
7+
required: false
8+
is-pypy:
9+
default: false
10+
environment-name:
11+
default: pandas-dev
12+
python-version:
13+
required: false
314
runs:
415
using: composite
516
steps:
6-
- name: Setting conda path
7-
run: echo "${HOME}/miniconda3/bin" >> $GITHUB_PATH
17+
- name: Set Arrow version in ${{ inputs.environment-file }} to ${{ inputs.pyarrow-version }}
18+
run: |
19+
grep -q ' - pyarrow' ${{ inputs.environment-file }}
20+
sed -i"" -e "s/ - pyarrow/ - pyarrow=${{ inputs.pyarrow-version }}/" ${{ inputs.environment-file }}
21+
cat ${{ inputs.environment-file }}
822
shell: bash -el {0}
23+
if: ${{ inputs.pyarrow-version }}
924

10-
- name: Setup environment and build pandas
11-
run: ci/setup_env.sh
25+
- name: Set Python version in ${{ inputs.environment-file }} to ${{ inputs.python-version }}
26+
run: |
27+
echo " - python=${{ inputs.pyarrow-version }}" >> ${{ inputs.environment-file }}
28+
cat ${{ inputs.environment-file }}
1229
shell: bash -el {0}
30+
if: ${{ inputs.python-version }}
31+
32+
- name: Pin setuptools (GH#44980)
33+
run: |
34+
echo ' - setuptools <60' >> ${{ inputs.environment-file }}
35+
shell: bash -el {0}
36+
37+
- name: Install ${{ inputs.environment-file }} (Python ${{ inputs.python-version }})
38+
uses: conda-incubator/[email protected]
39+
with:
40+
activate-environment: ${{ inputs.environment-name }}
41+
environment-file: ${{ inputs.environment-file }}
42+
channel-priority: strict
43+
channels: conda-forge
44+
mamba-version: "0.22"
45+
use-mamba: true
46+
if: ${{ inputs.is-pypy == 'false' }} # No pypy3.8 support
47+
48+
- name: Setup PyPy
49+
uses: actions/setup-python@v3
50+
with:
51+
python-version: "pypy-3.8"
52+
if: ${{ inputs.is-pypy == 'true' }}
53+
54+
- name: Setup PyPy dependencies
55+
# TODO: re-enable cov, its slowing the tests down though
56+
run: pip install Cython numpy python-dateutil pytz pytest>=6.0 pytest-xdist>=1.31.0 hypothesis>=5.5.3 pytest-asyncio>=0.17
57+
shell: bash -el {0}
58+
if: ${{ inputs.is-pypy == 'true' }}

‎.github/workflows/docbuild-and-upload.yml

+6-7
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,9 @@ jobs:
1818
web_and_docs:
1919
name: Doc Build and Upload
2020
runs-on: ubuntu-latest
21+
defaults:
22+
run:
23+
shell: bash -el {0}
2124

2225
concurrency:
2326
# https://github.community/t/concurrecy-not-work-for-push/183068/7
@@ -31,16 +34,12 @@ jobs:
3134
fetch-depth: 0
3235

3336
- name: Set up pandas
34-
uses: ./.github/actions/setup
37+
run: ci/setup_env.sh
3538

3639
- name: Build website
37-
run: |
38-
source activate pandas-dev
39-
python web/pandas_web.py web/pandas --target-path=web/build
40+
run: python web/pandas_web.py web/pandas --target-path=web/build
4041
- name: Build documentation
41-
run: |
42-
source activate pandas-dev
43-
doc/make.py --warnings-are-errors
42+
run: doc/make.py --warnings-are-errors
4443

4544
- name: Install ssh key
4645
run: |

‎.github/workflows/posix.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -140,7 +140,7 @@ jobs:
140140

141141
- uses: conda-incubator/[email protected]
142142
with:
143-
mamba-version: "*"
143+
mamba-version: "0.22"
144144
channels: conda-forge
145145
activate-environment: pandas-dev
146146
channel-priority: flexible

‎.github/workflows/sdist.yml

+47-38
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,8 @@ on:
1414
- "doc/**"
1515

1616
jobs:
17-
build:
18-
if: ${{ github.event.label.name == 'Build' || contains(github.event.pull_request.labels.*.name, 'Build') || github.event_name == 'push'}}
17+
sdist:
18+
#if: ${{ github.event.label.name == 'Build' || contains(github.event.pull_request.labels.*.name, 'Build') || github.event_name == 'push'}}
1919
runs-on: ubuntu-latest
2020
timeout-minutes: 60
2121
defaults:
@@ -25,67 +25,76 @@ jobs:
2525
strategy:
2626
fail-fast: false
2727
matrix:
28-
python-version: ["3.8", "3.9", "3.10"]
28+
# 'numpy-version' is oldest supported NumPy
29+
include:
30+
- python-version: "3.8"
31+
numpy-version: "1.18.5"
32+
- python-version: "3.9"
33+
numpy-version: "1.19.4"
34+
- python-version: "3.10"
35+
numpy-version: "1.21.3"
36+
37+
name: sdist Python ${{ matrix.python-version }} NumPy ${{ matrix.numpy-version }}
38+
2939
concurrency:
3040
# https://github.community/t/concurrecy-not-work-for-push/183068/7
31-
group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{matrix.python-version}}-sdist
41+
group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.python-version }}-sdist
3242
cancel-in-progress: true
3343

3444
steps:
3545
- uses: actions/checkout@v3
3646
with:
3747
fetch-depth: 0
3848

39-
- name: Set up Python
40-
uses: actions/setup-python@v3
49+
- name: Set up Python ${{ matrix.python-version }} and install dependencies
50+
uses: ./.github/actions/setup-python
4151
with:
4252
python-version: ${{ matrix.python-version }}
4353

44-
# TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941
45-
- name: Install dependencies
46-
run: |
47-
python -m pip install --upgrade pip "setuptools<60.0.0" wheel
48-
49-
# GH 39416
50-
pip install numpy
51-
5254
- name: Build pandas sdist
53-
run: |
54-
pip list
55-
python setup.py sdist --formats=gztar
55+
run: python setup.py sdist --formats=gztar
5656

5757
- name: Upload sdist artifact
5858
uses: actions/upload-artifact@v3
5959
with:
60-
name: ${{matrix.python-version}}-sdist.gz
60+
name: ${{ matrix.python-version }}-sdist.gz
6161
path: dist/*.gz
6262

63-
- uses: conda-incubator/[email protected]
63+
- name: Create environment.yml with Python ${{ matrix.python-version }}, NumPy ${{ matrix.numpy-version }}
64+
run: |
65+
cat >> tmp_environment.yml <<EOF
66+
name: pandas-sdist
67+
channels:
68+
- conda-forge
69+
dependencies:
70+
- python=${{ matrix.python-version }}
71+
- setuptools<60
72+
- numpy=${{ matrix.numpy-version }}
73+
EOF
74+
75+
- name: Install Conda enviroment
76+
uses: conda-incubator/[email protected]
6477
with:
6578
activate-environment: pandas-sdist
79+
environment-file: tmp_environment.yml
80+
channel-priority: strict
6681
channels: conda-forge
67-
python-version: '${{ matrix.python-version }}'
82+
mamba-version: "0.22"
83+
use-mamba: true
6884

69-
# TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941
7085
- name: Install pandas from sdist
7186
run: |
72-
python -m pip install --upgrade "setuptools<60.0.0"
87+
mamba info
88+
# Double check that we have the expected NumPy
7389
pip list
74-
python -m pip install dist/*.gz
75-
76-
- name: Force oldest supported NumPy
77-
run: |
78-
case "${{matrix.python-version}}" in
79-
3.8)
80-
pip install numpy==1.18.5 ;;
81-
3.9)
82-
pip install numpy==1.19.3 ;;
83-
3.10)
84-
pip install numpy==1.21.2 ;;
85-
esac
90+
pip list | egrep -i "numpy.+${{ matrix.numpy-version }}"
91+
time python -m pip install -v dist/*.gz
8692
87-
- name: Import pandas
93+
- name: Build Version
8894
run: |
89-
cd ..
90-
conda list
91-
python -c "import pandas; pandas.show_versions();"
95+
pushd /tmp
96+
python -c "import pandas; pandas.show_versions()"
97+
# Double check that we have the expected Python and NumPy
98+
python -c "import pandas; pandas.show_versions()" | egrep -i "python.+: ${{ matrix.python-version }}"
99+
python -c "import pandas; pandas.show_versions()" | egrep -i "numpy.+: ${{ matrix.numpy-version }}"
100+
popd

0 commit comments

Comments
 (0)