From 46a6ba6cb036ecf1eabc7117077a63751f872f80 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Sat, 26 Feb 2022 01:55:51 +0100 Subject: [PATCH] Backport PR #45902: Use Mamba in Azure CI --- ci/azure/posix.yml | 11 +++-- ci/azure/windows.yml | 14 ++++-- ci/setup_env.sh | 107 +++++++++++++++++++++---------------------- 3 files changed, 69 insertions(+), 63 deletions(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 6d33cd7e5f5d9..b599cfa40e89d 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -25,15 +25,20 @@ jobs: - script: echo '##vso[task.prependpath]$(HOME)/miniconda3/bin' displayName: 'Set conda path' + - script: rm /usr/local/miniconda/pkgs/cache/*.json + displayName: 'Workaround for mamba-org/mamba#488' + - script: ci/setup_env.sh displayName: 'Setup environment and build pandas' - script: | - source activate pandas-dev - ci/run_tests.sh + conda run -n pandas-dev --no-capture-output ci/run_tests.sh displayName: 'Test' - - script: source activate pandas-dev && pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd + - script: | + pushd /tmp + conda run -n pandas-dev python -c "import pandas; pandas.show_versions()" + popd displayName: 'Build versions' - task: PublishTestResults@2 diff --git a/ci/azure/windows.yml b/ci/azure/windows.yml index 3b489d92388da..02c6564579aa2 100644 --- a/ci/azure/windows.yml +++ b/ci/azure/windows.yml @@ -26,16 +26,22 @@ jobs: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts" Write-Host "##vso[task.prependpath]$HOME/miniconda3/bin" displayName: 'Add conda to PATH' - - script: conda update -q -n base conda - displayName: 'Update conda' + - bash: conda install -yv -c conda-forge -n base 'mamba>=0.21.2' + displayName: 'Install mamba' - bash: | - conda env create -q --file ci\\deps\\actions-$(CONDA_PY).yaml + # See https://github.com/mamba-org/mamba/issues/1370 + # See https://github.com/mamba-org/mamba/issues/633 + C:\\Miniconda\\condabin\\mamba.bat create -n pandas-dev + C:\\Miniconda\\condabin\\mamba.bat env update -n pandas-dev --file ci\\deps\\actions-$(CONDA_PY).yaml + # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 + C:\\Miniconda\\condabin\\mamba.bat install -n pandas-dev 'setuptools<60' + C:\\Miniconda\\condabin\\mamba.bat list -n pandas-dev displayName: 'Create anaconda environment' - bash: | source activate pandas-dev conda list - python setup.py build_ext -q -j 4 + python setup.py build_ext -q -j 2 python -m pip install --no-build-isolation -e . displayName: 'Build' - bash: | diff --git a/ci/setup_env.sh b/ci/setup_env.sh index d51ff98b241a6..5968fc0c0c88b 100755 --- a/ci/setup_env.sh +++ b/ci/setup_env.sh @@ -13,29 +13,34 @@ if [[ "$(uname)" == "Linux" && -n "$LC_ALL" ]]; then fi -echo "Install Miniconda" -DEFAULT_CONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest" -if [[ "$(uname -m)" == 'aarch64' ]]; then - CONDA_URL="https://github.com/conda-forge/miniforge/releases/download/4.10.1-4/Miniforge3-4.10.1-4-Linux-aarch64.sh" -elif [[ "$(uname)" == 'Linux' ]]; then - if [[ "$BITS32" == "yes" ]]; then - CONDA_URL="$DEFAULT_CONDA_URL-Linux-x86.sh" +MINICONDA_DIR=/usr/local/miniconda +if [ -e $MINICONDA_DIR ] && [ "$BITS32" != yes ]; then + echo "Found Miniconda installation at $MINICONDA_DIR" +else + echo "Install Miniconda" + DEFAULT_CONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest" + if [[ "$(uname -m)" == 'aarch64' ]]; then + CONDA_URL="https://github.com/conda-forge/miniforge/releases/download/4.10.1-4/Miniforge3-4.10.1-4-Linux-aarch64.sh" + elif [[ "$(uname)" == 'Linux' ]]; then + if [[ "$BITS32" == "yes" ]]; then + CONDA_URL="$DEFAULT_CONDA_URL-Linux-x86.sh" + else + CONDA_URL="$DEFAULT_CONDA_URL-Linux-x86_64.sh" + fi + elif [[ "$(uname)" == 'Darwin' ]]; then + CONDA_URL="$DEFAULT_CONDA_URL-MacOSX-x86_64.sh" else - CONDA_URL="$DEFAULT_CONDA_URL-Linux-x86_64.sh" + echo "OS $(uname) not supported" + exit 1 fi -elif [[ "$(uname)" == 'Darwin' ]]; then - CONDA_URL="$DEFAULT_CONDA_URL-MacOSX-x86_64.sh" -else - echo "OS $(uname) not supported" - exit 1 -fi -echo "Downloading $CONDA_URL" -wget -q $CONDA_URL -O miniconda.sh -chmod +x miniconda.sh + echo "Downloading $CONDA_URL" + wget -q $CONDA_URL -O miniconda.sh + chmod +x miniconda.sh -MINICONDA_DIR="$HOME/miniconda3" -rm -rf $MINICONDA_DIR -./miniconda.sh -b -p $MINICONDA_DIR + MINICONDA_DIR="$HOME/miniconda3" + rm -rf $MINICONDA_DIR + ./miniconda.sh -b -p $MINICONDA_DIR +fi export PATH=$MINICONDA_DIR/bin:$PATH echo @@ -46,25 +51,27 @@ echo echo "update conda" conda config --set ssl_verify false conda config --set quiet true --set always_yes true --set changeps1 false -conda install pip conda # create conda to create a historical artifact for pip & setuptools -conda update -n base conda -conda install -y -c conda-forge mamba +# TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 +conda install -y -c conda-forge -n base 'mamba>=0.21.2' pip echo "conda info -a" conda info -a -echo "source deactivate" -source deactivate - echo "conda list (root environment)" conda list +echo # Clean up any left-over from a previous build -conda remove --all -q -y -n pandas-dev +mamba env remove -n pandas-dev +echo "mamba env update --file=${ENV_FILE}" +# See https://github.com/mamba-org/mamba/issues/633 +mamba create -q -n pandas-dev +time mamba env update -n pandas-dev --file="${ENV_FILE}" +# TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 +mamba install -n pandas-dev 'setuptools<60' -echo -echo "mamba env create -q --file=${ENV_FILE}" -time mamba env create -q --file="${ENV_FILE}" +echo "conda list -n pandas-dev" +conda list -n pandas-dev if [[ "$BITS32" == "yes" ]]; then @@ -81,35 +88,23 @@ source activate pandas-dev # downstream CI jobs that may also build pandas from source. export PANDAS_CI=1 -echo -echo "remove any installed pandas package" -echo "w/o removing anything else" -conda remove pandas -y --force || true -pip uninstall -y pandas || true - -echo -echo "remove qt" -echo "causes problems with the clipboard, we use xsel for that" -conda remove qt -y --force || true - -echo -echo "conda list pandas" -conda list pandas - -# Make sure any error below is reported as such +if pip list | grep -q ^pandas; then + echo + echo "remove any installed pandas package w/o removing anything else" + pip uninstall -y pandas || true +fi -echo "[Build extensions]" -python setup.py build_ext -q -j2 +if [ "$(conda list -f qt --json)" != [] ]; then + echo + echo "remove qt" + echo "causes problems with the clipboard, we use xsel for that" + conda remove qt -y --force || true +fi -echo "[Updating pip]" -# TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 -python -m pip install --no-deps -U pip wheel "setuptools<60.0.0" +echo "Build extensions" +python setup.py build_ext -q -j3 -echo "[Install pandas]" +echo "Install pandas" python -m pip install --no-build-isolation -e . -echo -echo "conda list" -conda list - echo "done"