Skip to content

Use Mamba in Azure CI #45902

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Feb 26, 2022
Merged

Use Mamba in Azure CI #45902

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
42e7966
Use Mamba in Azure Pipelines
jonashaag Feb 11, 2022
bdf538f
Fix setuptools < 60 on ARM
jonashaag Feb 21, 2022
f1b69f8
Pin setuptools < 60 on Windows
jonashaag Feb 21, 2022
a4705c1
Use more recent Mamba
jonashaag Feb 21, 2022
9ae6bce
Pin setuptools < 60
jonashaag Feb 21, 2022
3c5bb13
Pin setuptools < 60
jonashaag Feb 21, 2022
1be9bf4
Workaround for https://github.com/mamba-org/mamba/issues/488
jonashaag Feb 22, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 8 additions & 3 deletions ci/azure/posix.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,15 +25,20 @@ jobs:
- script: echo '##vso[task.prependpath]$(HOME)/miniconda3/bin'
displayName: 'Set conda path'

- script: rm /usr/local/miniconda/pkgs/cache/*.json
displayName: 'Workaround for mamba-org/mamba#488'

- script: ci/setup_env.sh
displayName: 'Setup environment and build pandas'

- script: |
source activate pandas-dev
ci/run_tests.sh
conda run -n pandas-dev --no-capture-output ci/run_tests.sh
displayName: 'Test'

- script: source activate pandas-dev && pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd
- script: |
pushd /tmp
conda run -n pandas-dev python -c "import pandas; pandas.show_versions()"
popd
displayName: 'Build versions'

- task: PublishTestResults@2
Expand Down
14 changes: 10 additions & 4 deletions ci/azure/windows.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,16 +26,22 @@ jobs:
Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
Write-Host "##vso[task.prependpath]$HOME/miniconda3/bin"
displayName: 'Add conda to PATH'
- script: conda update -q -n base conda
displayName: 'Update conda'
- bash: conda install -yv -c conda-forge -n base 'mamba>=0.21.2'
displayName: 'Install mamba'

- bash: |
conda env create -q --file ci\\deps\\actions-$(CONDA_PY).yaml
# See https://github.com/mamba-org/mamba/issues/1370
# See https://github.com/mamba-org/mamba/issues/633
C:\\Miniconda\\condabin\\mamba.bat create -n pandas-dev
C:\\Miniconda\\condabin\\mamba.bat env update -n pandas-dev --file ci\\deps\\actions-$(CONDA_PY).yaml
# TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941
C:\\Miniconda\\condabin\\mamba.bat install -n pandas-dev 'setuptools<60'
C:\\Miniconda\\condabin\\mamba.bat list -n pandas-dev
displayName: 'Create anaconda environment'
- bash: |
source activate pandas-dev
conda list
python setup.py build_ext -q -j 4
python setup.py build_ext -q -j 2
python -m pip install --no-build-isolation -e .
displayName: 'Build'
- bash: |
Expand Down
107 changes: 51 additions & 56 deletions ci/setup_env.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,29 +13,34 @@ if [[ "$(uname)" == "Linux" && -n "$LC_ALL" ]]; then
fi


echo "Install Miniconda"
DEFAULT_CONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest"
if [[ "$(uname -m)" == 'aarch64' ]]; then
CONDA_URL="https://github.com/conda-forge/miniforge/releases/download/4.10.1-4/Miniforge3-4.10.1-4-Linux-aarch64.sh"
elif [[ "$(uname)" == 'Linux' ]]; then
if [[ "$BITS32" == "yes" ]]; then
CONDA_URL="$DEFAULT_CONDA_URL-Linux-x86.sh"
MINICONDA_DIR=/usr/local/miniconda
if [ -e $MINICONDA_DIR ] && [ "$BITS32" != yes ]; then
echo "Found Miniconda installation at $MINICONDA_DIR"
else
echo "Install Miniconda"
DEFAULT_CONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest"
if [[ "$(uname -m)" == 'aarch64' ]]; then
CONDA_URL="https://github.com/conda-forge/miniforge/releases/download/4.10.1-4/Miniforge3-4.10.1-4-Linux-aarch64.sh"
elif [[ "$(uname)" == 'Linux' ]]; then
if [[ "$BITS32" == "yes" ]]; then
CONDA_URL="$DEFAULT_CONDA_URL-Linux-x86.sh"
else
CONDA_URL="$DEFAULT_CONDA_URL-Linux-x86_64.sh"
fi
elif [[ "$(uname)" == 'Darwin' ]]; then
CONDA_URL="$DEFAULT_CONDA_URL-MacOSX-x86_64.sh"
else
CONDA_URL="$DEFAULT_CONDA_URL-Linux-x86_64.sh"
echo "OS $(uname) not supported"
exit 1
fi
elif [[ "$(uname)" == 'Darwin' ]]; then
CONDA_URL="$DEFAULT_CONDA_URL-MacOSX-x86_64.sh"
else
echo "OS $(uname) not supported"
exit 1
fi
echo "Downloading $CONDA_URL"
wget -q $CONDA_URL -O miniconda.sh
chmod +x miniconda.sh
echo "Downloading $CONDA_URL"
wget -q $CONDA_URL -O miniconda.sh
chmod +x miniconda.sh

MINICONDA_DIR="$HOME/miniconda3"
rm -rf $MINICONDA_DIR
./miniconda.sh -b -p $MINICONDA_DIR
MINICONDA_DIR="$HOME/miniconda3"
rm -rf $MINICONDA_DIR
./miniconda.sh -b -p $MINICONDA_DIR
fi
export PATH=$MINICONDA_DIR/bin:$PATH

echo
Expand All @@ -46,25 +51,27 @@ echo
echo "update conda"
conda config --set ssl_verify false
conda config --set quiet true --set always_yes true --set changeps1 false
conda install pip conda # create conda to create a historical artifact for pip & setuptools
conda update -n base conda
conda install -y -c conda-forge mamba
# TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941
conda install -y -c conda-forge -n base 'mamba>=0.21.2' pip

echo "conda info -a"
conda info -a

echo "source deactivate"
source deactivate

echo "conda list (root environment)"
conda list

echo
# Clean up any left-over from a previous build
conda remove --all -q -y -n pandas-dev
mamba env remove -n pandas-dev
echo "mamba env update --file=${ENV_FILE}"
# See https://github.com/mamba-org/mamba/issues/633
mamba create -q -n pandas-dev
time mamba env update -n pandas-dev --file="${ENV_FILE}"
# TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941
mamba install -n pandas-dev 'setuptools<60'

echo
echo "mamba env create -q --file=${ENV_FILE}"
time mamba env create -q --file="${ENV_FILE}"
echo "conda list -n pandas-dev"
conda list -n pandas-dev


if [[ "$BITS32" == "yes" ]]; then
Expand All @@ -81,35 +88,23 @@ source activate pandas-dev
# downstream CI jobs that may also build pandas from source.
export PANDAS_CI=1

echo
echo "remove any installed pandas package"
echo "w/o removing anything else"
conda remove pandas -y --force || true
pip uninstall -y pandas || true

echo
echo "remove qt"
echo "causes problems with the clipboard, we use xsel for that"
conda remove qt -y --force || true

echo
echo "conda list pandas"
conda list pandas

# Make sure any error below is reported as such
if pip list | grep -q ^pandas; then
echo
echo "remove any installed pandas package w/o removing anything else"
pip uninstall -y pandas || true
fi

echo "[Build extensions]"
python setup.py build_ext -q -j2
if [ "$(conda list -f qt --json)" != [] ]; then
echo
echo "remove qt"
echo "causes problems with the clipboard, we use xsel for that"
conda remove qt -y --force || true
fi

echo "[Updating pip]"
# TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941
python -m pip install --no-deps -U pip wheel "setuptools<60.0.0"
echo "Build extensions"
python setup.py build_ext -q -j3

echo "[Install pandas]"
echo "Install pandas"
python -m pip install --no-build-isolation -e .

echo
echo "conda list"
conda list

echo "done"