Skip to content

Commit 40e63c4

Browse files
jonashaagjonashaag
authored
Use Mamba in Azure CI (#45902)
1 parent d9147eb commit 40e63c4

File tree

3 files changed

+69
-63
lines changed

3 files changed

+69
-63
lines changed

‎ci/azure/posix.yml

+8-3
Original file line numberDiff line numberDiff line change
@@ -25,15 +25,20 @@ jobs:
2525
- script: echo '##vso[task.prependpath]$(HOME)/miniconda3/bin'
2626
displayName: 'Set conda path'
2727

28+
- script: rm /usr/local/miniconda/pkgs/cache/*.json
29+
displayName: 'Workaround for mamba-org/mamba#488'
30+
2831
- script: ci/setup_env.sh
2932
displayName: 'Setup environment and build pandas'
3033

3134
- script: |
32-
source activate pandas-dev
33-
ci/run_tests.sh
35+
conda run -n pandas-dev --no-capture-output ci/run_tests.sh
3436
displayName: 'Test'
3537
36-
- script: source activate pandas-dev && pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd
38+
- script: |
39+
pushd /tmp
40+
conda run -n pandas-dev python -c "import pandas; pandas.show_versions()"
41+
popd
3742
displayName: 'Build versions'
3843
3944
- task: PublishTestResults@2

‎ci/azure/windows.yml

+10-4
Original file line numberDiff line numberDiff line change
@@ -26,16 +26,22 @@ jobs:
2626
Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
2727
Write-Host "##vso[task.prependpath]$HOME/miniconda3/bin"
2828
displayName: 'Add conda to PATH'
29-
- script: conda update -q -n base conda
30-
displayName: 'Update conda'
29+
- bash: conda install -yv -c conda-forge -n base 'mamba>=0.21.2'
30+
displayName: 'Install mamba'
3131

3232
- bash: |
33-
conda env create -q --file ci\\deps\\actions-$(CONDA_PY).yaml
33+
# See https://github.com/mamba-org/mamba/issues/1370
34+
# See https://github.com/mamba-org/mamba/issues/633
35+
C:\\Miniconda\\condabin\\mamba.bat create -n pandas-dev
36+
C:\\Miniconda\\condabin\\mamba.bat env update -n pandas-dev --file ci\\deps\\actions-$(CONDA_PY).yaml
37+
# TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941
38+
C:\\Miniconda\\condabin\\mamba.bat install -n pandas-dev 'setuptools<60'
39+
C:\\Miniconda\\condabin\\mamba.bat list -n pandas-dev
3440
displayName: 'Create anaconda environment'
3541
- bash: |
3642
source activate pandas-dev
3743
conda list
38-
python setup.py build_ext -q -j 4
44+
python setup.py build_ext -q -j 2
3945
python -m pip install --no-build-isolation -e .
4046
displayName: 'Build'
4147
- bash: |

‎ci/setup_env.sh

+51-56
Original file line numberDiff line numberDiff line change
@@ -13,29 +13,34 @@ if [[ "$(uname)" == "Linux" && -n "$LC_ALL" ]]; then
1313
fi
1414

1515

16-
echo "Install Miniconda"
17-
DEFAULT_CONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest"
18-
if [[ "$(uname -m)" == 'aarch64' ]]; then
19-
CONDA_URL="https://github.com/conda-forge/miniforge/releases/download/4.10.1-4/Miniforge3-4.10.1-4-Linux-aarch64.sh"
20-
elif [[ "$(uname)" == 'Linux' ]]; then
21-
if [[ "$BITS32" == "yes" ]]; then
22-
CONDA_URL="$DEFAULT_CONDA_URL-Linux-x86.sh"
16+
MINICONDA_DIR=/usr/local/miniconda
17+
if [ -e $MINICONDA_DIR ] && [ "$BITS32" != yes ]; then
18+
echo "Found Miniconda installation at $MINICONDA_DIR"
19+
else
20+
echo "Install Miniconda"
21+
DEFAULT_CONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest"
22+
if [[ "$(uname -m)" == 'aarch64' ]]; then
23+
CONDA_URL="https://github.com/conda-forge/miniforge/releases/download/4.10.1-4/Miniforge3-4.10.1-4-Linux-aarch64.sh"
24+
elif [[ "$(uname)" == 'Linux' ]]; then
25+
if [[ "$BITS32" == "yes" ]]; then
26+
CONDA_URL="$DEFAULT_CONDA_URL-Linux-x86.sh"
27+
else
28+
CONDA_URL="$DEFAULT_CONDA_URL-Linux-x86_64.sh"
29+
fi
30+
elif [[ "$(uname)" == 'Darwin' ]]; then
31+
CONDA_URL="$DEFAULT_CONDA_URL-MacOSX-x86_64.sh"
2332
else
24-
CONDA_URL="$DEFAULT_CONDA_URL-Linux-x86_64.sh"
33+
echo "OS $(uname) not supported"
34+
exit 1
2535
fi
26-
elif [[ "$(uname)" == 'Darwin' ]]; then
27-
CONDA_URL="$DEFAULT_CONDA_URL-MacOSX-x86_64.sh"
28-
else
29-
echo "OS $(uname) not supported"
30-
exit 1
31-
fi
32-
echo "Downloading $CONDA_URL"
33-
wget -q $CONDA_URL -O miniconda.sh
34-
chmod +x miniconda.sh
36+
echo "Downloading $CONDA_URL"
37+
wget -q $CONDA_URL -O miniconda.sh
38+
chmod +x miniconda.sh
3539

36-
MINICONDA_DIR="$HOME/miniconda3"
37-
rm -rf $MINICONDA_DIR
38-
./miniconda.sh -b -p $MINICONDA_DIR
40+
MINICONDA_DIR="$HOME/miniconda3"
41+
rm -rf $MINICONDA_DIR
42+
./miniconda.sh -b -p $MINICONDA_DIR
43+
fi
3944
export PATH=$MINICONDA_DIR/bin:$PATH
4045

4146
echo
@@ -46,25 +51,27 @@ echo
4651
echo "update conda"
4752
conda config --set ssl_verify false
4853
conda config --set quiet true --set always_yes true --set changeps1 false
49-
conda install pip conda # create conda to create a historical artifact for pip & setuptools
50-
conda update -n base conda
51-
conda install -y -c conda-forge mamba
54+
# TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941
55+
conda install -y -c conda-forge -n base 'mamba>=0.21.2' pip
5256

5357
echo "conda info -a"
5458
conda info -a
5559

56-
echo "source deactivate"
57-
source deactivate
58-
5960
echo "conda list (root environment)"
6061
conda list
6162

63+
echo
6264
# Clean up any left-over from a previous build
63-
conda remove --all -q -y -n pandas-dev
65+
mamba env remove -n pandas-dev
66+
echo "mamba env update --file=${ENV_FILE}"
67+
# See https://github.com/mamba-org/mamba/issues/633
68+
mamba create -q -n pandas-dev
69+
time mamba env update -n pandas-dev --file="${ENV_FILE}"
70+
# TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941
71+
mamba install -n pandas-dev 'setuptools<60'
6472

65-
echo
66-
echo "mamba env create -q --file=${ENV_FILE}"
67-
time mamba env create -q --file="${ENV_FILE}"
73+
echo "conda list -n pandas-dev"
74+
conda list -n pandas-dev
6875

6976
# From pyarrow on MacOS
7077
# ImportError: 2): Library not loaded: @rpath/libssl.1.1.dylib
@@ -89,35 +96,23 @@ source activate pandas-dev
8996
# downstream CI jobs that may also build pandas from source.
9097
export PANDAS_CI=1
9198

92-
echo
93-
echo "remove any installed pandas package"
94-
echo "w/o removing anything else"
95-
conda remove pandas -y --force || true
96-
pip uninstall -y pandas || true
97-
98-
echo
99-
echo "remove qt"
100-
echo "causes problems with the clipboard, we use xsel for that"
101-
conda remove qt -y --force || true
102-
103-
echo
104-
echo "conda list pandas"
105-
conda list pandas
106-
107-
# Make sure any error below is reported as such
99+
if pip list | grep -q ^pandas; then
100+
echo
101+
echo "remove any installed pandas package w/o removing anything else"
102+
pip uninstall -y pandas || true
103+
fi
108104

109-
echo "[Build extensions]"
110-
python setup.py build_ext -q -j2
105+
if [ "$(conda list -f qt --json)" != [] ]; then
106+
echo
107+
echo "remove qt"
108+
echo "causes problems with the clipboard, we use xsel for that"
109+
conda remove qt -y --force || true
110+
fi
111111

112-
echo "[Updating pip]"
113-
# TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941
114-
python -m pip install --no-deps -U pip wheel "setuptools<60.0.0"
112+
echo "Build extensions"
113+
python setup.py build_ext -q -j3
115114

116-
echo "[Install pandas]"
115+
echo "Install pandas"
117116
python -m pip install --no-build-isolation -e .
118117

119-
echo
120-
echo "conda list"
121-
conda list
122-
123118
echo "done"

0 commit comments

Comments
 (0)