From 42e79665a526a79c0a75316861b79a3eaa3b71c0 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Feb 2022 16:14:58 +0100 Subject: [PATCH 1/7] Use Mamba in Azure Pipelines Also clean up ci/setup_env.sh --- ci/azure/posix.yml | 8 ++-- ci/azure/windows.yml | 12 +++-- ci/setup_env.sh | 105 ++++++++++++++++++++----------------------- 3 files changed, 62 insertions(+), 63 deletions(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 6d33cd7e5f5d9..17ca20897d1ee 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -29,11 +29,13 @@ jobs: displayName: 'Setup environment and build pandas' - script: | - source activate pandas-dev - ci/run_tests.sh + conda run -n pandas-dev --no-capture-output ci/run_tests.sh displayName: 'Test' - - script: source activate pandas-dev && pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd + - script: | + pushd /tmp + conda run -n pandas-dev python -c "import pandas; pandas.show_versions()" + popd displayName: 'Build versions' - task: PublishTestResults@2 diff --git a/ci/azure/windows.yml b/ci/azure/windows.yml index 3b489d92388da..ba2128c5b7ca3 100644 --- a/ci/azure/windows.yml +++ b/ci/azure/windows.yml @@ -26,16 +26,20 @@ jobs: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts" Write-Host "##vso[task.prependpath]$HOME/miniconda3/bin" displayName: 'Add conda to PATH' - - script: conda update -q -n base conda - displayName: 'Update conda' + - bash: conda install -yv -c conda-forge -n base 'mamba<0.21' + displayName: 'Install mamba' - bash: | - conda env create -q --file ci\\deps\\actions-$(CONDA_PY).yaml + # See https://github.com/mamba-org/mamba/issues/1370 + # See https://github.com/mamba-org/mamba/issues/633 + C:\\Miniconda\\condabin\\mamba.bat create -n pandas-dev + C:\\Miniconda\\condabin\\mamba.bat env update -n pandas-dev --file ci\\deps\\actions-$(CONDA_PY).yaml + C:\\Miniconda\\condabin\\mamba.bat list -n pandas-dev displayName: 'Create anaconda environment' - bash: | source activate pandas-dev conda list - python setup.py build_ext -q -j 4 + python setup.py build_ext -q -j 2 python -m pip install --no-build-isolation -e . displayName: 'Build' - bash: | diff --git a/ci/setup_env.sh b/ci/setup_env.sh index d51ff98b241a6..679c8056ec6c4 100755 --- a/ci/setup_env.sh +++ b/ci/setup_env.sh @@ -13,29 +13,34 @@ if [[ "$(uname)" == "Linux" && -n "$LC_ALL" ]]; then fi -echo "Install Miniconda" -DEFAULT_CONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest" -if [[ "$(uname -m)" == 'aarch64' ]]; then - CONDA_URL="https://github.com/conda-forge/miniforge/releases/download/4.10.1-4/Miniforge3-4.10.1-4-Linux-aarch64.sh" -elif [[ "$(uname)" == 'Linux' ]]; then - if [[ "$BITS32" == "yes" ]]; then - CONDA_URL="$DEFAULT_CONDA_URL-Linux-x86.sh" +MINICONDA_DIR=/usr/local/miniconda +if [ -e $MINICONDA_DIR ] && [ "$BITS32" != yes ]; then + echo "Found Miniconda installation at $MINICONDA_DIR" +else + echo "Install Miniconda" + DEFAULT_CONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest" + if [[ "$(uname -m)" == 'aarch64' ]]; then + CONDA_URL="https://github.com/conda-forge/miniforge/releases/download/4.10.1-4/Miniforge3-4.10.1-4-Linux-aarch64.sh" + elif [[ "$(uname)" == 'Linux' ]]; then + if [[ "$BITS32" == "yes" ]]; then + CONDA_URL="$DEFAULT_CONDA_URL-Linux-x86.sh" + else + CONDA_URL="$DEFAULT_CONDA_URL-Linux-x86_64.sh" + fi + elif [[ "$(uname)" == 'Darwin' ]]; then + CONDA_URL="$DEFAULT_CONDA_URL-MacOSX-x86_64.sh" else - CONDA_URL="$DEFAULT_CONDA_URL-Linux-x86_64.sh" + echo "OS $(uname) not supported" + exit 1 fi -elif [[ "$(uname)" == 'Darwin' ]]; then - CONDA_URL="$DEFAULT_CONDA_URL-MacOSX-x86_64.sh" -else - echo "OS $(uname) not supported" - exit 1 -fi -echo "Downloading $CONDA_URL" -wget -q $CONDA_URL -O miniconda.sh -chmod +x miniconda.sh + echo "Downloading $CONDA_URL" + wget -q $CONDA_URL -O miniconda.sh + chmod +x miniconda.sh -MINICONDA_DIR="$HOME/miniconda3" -rm -rf $MINICONDA_DIR -./miniconda.sh -b -p $MINICONDA_DIR + MINICONDA_DIR="$HOME/miniconda3" + rm -rf $MINICONDA_DIR + ./miniconda.sh -b -p $MINICONDA_DIR +fi export PATH=$MINICONDA_DIR/bin:$PATH echo @@ -46,25 +51,25 @@ echo echo "update conda" conda config --set ssl_verify false conda config --set quiet true --set always_yes true --set changeps1 false -conda install pip conda # create conda to create a historical artifact for pip & setuptools -conda update -n base conda -conda install -y -c conda-forge mamba +# TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 +conda install -y -c conda-forge -n base mamba pip 'setuptools<60' echo "conda info -a" conda info -a -echo "source deactivate" -source deactivate - echo "conda list (root environment)" conda list +echo # Clean up any left-over from a previous build -conda remove --all -q -y -n pandas-dev +mamba env remove -n pandas-dev +echo "mamba env update --file=${ENV_FILE}" +# See https://github.com/mamba-org/mamba/issues/633 +mamba create -q -n pandas-dev +time mamba env update -n pandas-dev --file="${ENV_FILE}" -echo -echo "mamba env create -q --file=${ENV_FILE}" -time mamba env create -q --file="${ENV_FILE}" +echo "conda list -n pandas-dev" +conda list -n pandas-dev if [[ "$BITS32" == "yes" ]]; then @@ -81,35 +86,23 @@ source activate pandas-dev # downstream CI jobs that may also build pandas from source. export PANDAS_CI=1 -echo -echo "remove any installed pandas package" -echo "w/o removing anything else" -conda remove pandas -y --force || true -pip uninstall -y pandas || true - -echo -echo "remove qt" -echo "causes problems with the clipboard, we use xsel for that" -conda remove qt -y --force || true - -echo -echo "conda list pandas" -conda list pandas - -# Make sure any error below is reported as such +if pip list | grep -q ^pandas; then + echo + echo "remove any installed pandas package w/o removing anything else" + pip uninstall -y pandas || true +fi -echo "[Build extensions]" -python setup.py build_ext -q -j2 +if [ "$(conda list -f qt --json)" != [] ]; then + echo + echo "remove qt" + echo "causes problems with the clipboard, we use xsel for that" + conda remove qt -y --force || true +fi -echo "[Updating pip]" -# TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 -python -m pip install --no-deps -U pip wheel "setuptools<60.0.0" +echo "Build extensions" +python setup.py build_ext -q -j3 -echo "[Install pandas]" +echo "Install pandas" python -m pip install --no-build-isolation -e . -echo -echo "conda list" -conda list - echo "done" From bdf538f7dcbe03a97617c0644a263c760f6ef8ca Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Mon, 21 Feb 2022 22:09:09 +0100 Subject: [PATCH 2/7] Fix setuptools < 60 on ARM --- ci/setup_env.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/setup_env.sh b/ci/setup_env.sh index 679c8056ec6c4..9383b5cc931c1 100755 --- a/ci/setup_env.sh +++ b/ci/setup_env.sh @@ -65,7 +65,8 @@ echo mamba env remove -n pandas-dev echo "mamba env update --file=${ENV_FILE}" # See https://github.com/mamba-org/mamba/issues/633 -mamba create -q -n pandas-dev +# TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 +mamba create -q -n pandas-dev 'setuptools<60' time mamba env update -n pandas-dev --file="${ENV_FILE}" echo "conda list -n pandas-dev" From f1b69f8c46b7c6e947f5cc50dddcc5cf99ecbb3a Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Mon, 21 Feb 2022 22:25:55 +0100 Subject: [PATCH 3/7] Pin setuptools < 60 on Windows --- ci/azure/windows.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ci/azure/windows.yml b/ci/azure/windows.yml index ba2128c5b7ca3..d3d601368e602 100644 --- a/ci/azure/windows.yml +++ b/ci/azure/windows.yml @@ -26,13 +26,14 @@ jobs: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts" Write-Host "##vso[task.prependpath]$HOME/miniconda3/bin" displayName: 'Add conda to PATH' - - bash: conda install -yv -c conda-forge -n base 'mamba<0.21' + - bash: conda install -yv -c conda-forge -n base 'mamba>=0.21.2' displayName: 'Install mamba' - bash: | # See https://github.com/mamba-org/mamba/issues/1370 # See https://github.com/mamba-org/mamba/issues/633 - C:\\Miniconda\\condabin\\mamba.bat create -n pandas-dev + # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 + C:\\Miniconda\\condabin\\mamba.bat create -n pandas-dev 'setuptools<60' C:\\Miniconda\\condabin\\mamba.bat env update -n pandas-dev --file ci\\deps\\actions-$(CONDA_PY).yaml C:\\Miniconda\\condabin\\mamba.bat list -n pandas-dev displayName: 'Create anaconda environment' From a4705c1fd7488f121dc1002ddd7365bf6c58f451 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Mon, 21 Feb 2022 22:26:22 +0100 Subject: [PATCH 4/7] Use more recent Mamba --- ci/setup_env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/setup_env.sh b/ci/setup_env.sh index 9383b5cc931c1..2fc7969ab9dc0 100755 --- a/ci/setup_env.sh +++ b/ci/setup_env.sh @@ -52,7 +52,7 @@ echo "update conda" conda config --set ssl_verify false conda config --set quiet true --set always_yes true --set changeps1 false # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 -conda install -y -c conda-forge -n base mamba pip 'setuptools<60' +conda install -y -c conda-forge -n base 'mamba>=0.21.2' pip 'setuptools<60' echo "conda info -a" conda info -a From 9ae6bce0376969f4041a1c6f6bf9396652cfb304 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Mon, 21 Feb 2022 22:39:09 +0100 Subject: [PATCH 5/7] Pin setuptools < 60 --- ci/azure/windows.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ci/azure/windows.yml b/ci/azure/windows.yml index d3d601368e602..02c6564579aa2 100644 --- a/ci/azure/windows.yml +++ b/ci/azure/windows.yml @@ -32,9 +32,10 @@ jobs: - bash: | # See https://github.com/mamba-org/mamba/issues/1370 # See https://github.com/mamba-org/mamba/issues/633 - # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 - C:\\Miniconda\\condabin\\mamba.bat create -n pandas-dev 'setuptools<60' + C:\\Miniconda\\condabin\\mamba.bat create -n pandas-dev C:\\Miniconda\\condabin\\mamba.bat env update -n pandas-dev --file ci\\deps\\actions-$(CONDA_PY).yaml + # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 + C:\\Miniconda\\condabin\\mamba.bat install -n pandas-dev 'setuptools<60' C:\\Miniconda\\condabin\\mamba.bat list -n pandas-dev displayName: 'Create anaconda environment' - bash: | From 3c5bb13498697f2ab3c89dabfe60f6e578ee2676 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Mon, 21 Feb 2022 22:40:21 +0100 Subject: [PATCH 6/7] Pin setuptools < 60 --- ci/setup_env.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ci/setup_env.sh b/ci/setup_env.sh index 2fc7969ab9dc0..5968fc0c0c88b 100755 --- a/ci/setup_env.sh +++ b/ci/setup_env.sh @@ -52,7 +52,7 @@ echo "update conda" conda config --set ssl_verify false conda config --set quiet true --set always_yes true --set changeps1 false # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 -conda install -y -c conda-forge -n base 'mamba>=0.21.2' pip 'setuptools<60' +conda install -y -c conda-forge -n base 'mamba>=0.21.2' pip echo "conda info -a" conda info -a @@ -65,9 +65,10 @@ echo mamba env remove -n pandas-dev echo "mamba env update --file=${ENV_FILE}" # See https://github.com/mamba-org/mamba/issues/633 -# TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 -mamba create -q -n pandas-dev 'setuptools<60' +mamba create -q -n pandas-dev time mamba env update -n pandas-dev --file="${ENV_FILE}" +# TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 +mamba install -n pandas-dev 'setuptools<60' echo "conda list -n pandas-dev" conda list -n pandas-dev From 1be9bf489f2086dfc82e82a2c5a20d73a5ad0a50 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Tue, 22 Feb 2022 22:13:29 +0100 Subject: [PATCH 7/7] Workaround for https://github.com/mamba-org/mamba/issues/488 --- ci/azure/posix.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 17ca20897d1ee..b599cfa40e89d 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -25,6 +25,9 @@ jobs: - script: echo '##vso[task.prependpath]$(HOME)/miniconda3/bin' displayName: 'Set conda path' + - script: rm /usr/local/miniconda/pkgs/cache/*.json + displayName: 'Workaround for mamba-org/mamba#488' + - script: ci/setup_env.sh displayName: 'Setup environment and build pandas'