From 39fa0b05b91eed38bf91dc67a001c263a983e0ab Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Thu, 14 Oct 2021 11:57:28 +0100 Subject: [PATCH 01/27] BLD: Add conda cache to azure ci --- azure-pipelines.yml | 1 + ci/azure/posix.yml | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 6c685d09ab55a..4ea05fe8fd637 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -18,6 +18,7 @@ pr: variables: PYTEST_WORKERS: auto PYTEST_TARGET: pandas + CONDA_CACHE_DIR: $(Pipeline.Workspace)/.condarc jobs: # Mac and Linux use the same template diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index eaac17d50c315..cec7addfa59c0 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -24,6 +24,16 @@ jobs: - script: echo '##vso[task.prependpath]$(HOME)/miniconda3/bin' displayName: 'Set conda path' + - task: Cache@2 + displayName: Use cached Anaconda environment + inputs: + key: 'conda | "$(Agent.OS)" | "$(ENV_FILE)"' + restoreKeys: | + python | "$(Agent.OS)" + python + path: $(CONDA_CACHE_DIR) + cacheHitVar: CONDA_CACHE_RESTORED + - script: ci/setup_env.sh displayName: 'Setup environment and build pandas' From 2198ca89a9ca0db15a2142fd4d7e4343a00e50ee Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Thu, 14 Oct 2021 11:57:58 +0100 Subject: [PATCH 02/27] BLD: Add conda cache to Windows azure CI --- ci/azure/windows.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/ci/azure/windows.yml b/ci/azure/windows.yml index 3bd20b1399be2..9ef5a191ace37 100644 --- a/ci/azure/windows.yml +++ b/ci/azure/windows.yml @@ -43,10 +43,19 @@ jobs: displayName: 'Add conda to PATH' - script: conda update -q -n base conda displayName: 'Update conda' - + - task: Cache@2 + displayName: Use cached Anaconda environment + inputs: + key: 'conda | "$(Agent.OS)" | "$(ENV_FILE)"' + restoreKeys: | + python | "$(Agent.OS)" + python + path: $(CONDA_CACHE_DIR) + cacheHitVar: CONDA_CACHE_RESTORED - bash: | conda env create -q --file ci\\deps\\azure-windows-$(CONDA_PY).yaml displayName: 'Create anaconda environment' + condition: eq(variables.CONDA_CACHE_RESTORED, 'false') - bash: | source activate pandas-dev conda list From bba2735422b663ddb963e4f449a1d3bdb63a6d85 Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Thu, 14 Oct 2021 12:16:38 +0100 Subject: [PATCH 03/27] BLD: Trivial change to test cache --- ci/setup_env.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/setup_env.sh b/ci/setup_env.sh index 2e16bc6545161..a6759797b5bb4 100755 --- a/ci/setup_env.sh +++ b/ci/setup_env.sh @@ -26,8 +26,8 @@ elif [[ "$(uname)" == 'Linux' ]]; then elif [[ "$(uname)" == 'Darwin' ]]; then CONDA_URL="$DEFAULT_CONDA_URL-MacOSX-x86_64.sh" else - echo "OS $(uname) not supported" - exit 1 + echo "OS $(uname) not supported" + exit 1 fi echo "Downloading $CONDA_URL" wget -q $CONDA_URL -O miniconda.sh From 7dc9abeebfa696a3a245bd050c69634e4b9f0945 Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Mon, 25 Oct 2021 16:22:58 +0100 Subject: [PATCH 04/27] ENH: Add conda cache - windows and posix --- azure-pipelines.yml | 1 - ci/azure/posix.yml | 8 +++-- ci/azure/windows.yml | 3 +- ci/setup_conda.sh | 61 ++++++++++++++++++++++++++++++++++++++ ci/setup_env.sh | 70 +++----------------------------------------- 5 files changed, 73 insertions(+), 70 deletions(-) create mode 100755 ci/setup_conda.sh diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 4ea05fe8fd637..6c685d09ab55a 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -18,7 +18,6 @@ pr: variables: PYTEST_WORKERS: auto PYTEST_TARGET: pandas - CONDA_CACHE_DIR: $(Pipeline.Workspace)/.condarc jobs: # Mac and Linux use the same template diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index cec7addfa59c0..caed8b1717c92 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -28,14 +28,18 @@ jobs: displayName: Use cached Anaconda environment inputs: key: 'conda | "$(Agent.OS)" | "$(ENV_FILE)"' + path: $(HOME)/miniconda3 restoreKeys: | python | "$(Agent.OS)" python - path: $(CONDA_CACHE_DIR) cacheHitVar: CONDA_CACHE_RESTORED + - script: ci/setup_conda.sh + displayName: 'Install Miniconda in CI env' + - script: ci/setup_env.sh - displayName: 'Setup environment and build pandas' + displayName: 'Install conda dependencies and build pandas' + condition: eq(variables.CONDA_CACHE_RESTORED, 'false') - script: | source activate pandas-dev diff --git a/ci/azure/windows.yml b/ci/azure/windows.yml index 9ef5a191ace37..34a0f4dc1c005 100644 --- a/ci/azure/windows.yml +++ b/ci/azure/windows.yml @@ -43,6 +43,7 @@ jobs: displayName: 'Add conda to PATH' - script: conda update -q -n base conda displayName: 'Update conda' + - task: Cache@2 displayName: Use cached Anaconda environment inputs: @@ -50,7 +51,7 @@ jobs: restoreKeys: | python | "$(Agent.OS)" python - path: $(CONDA_CACHE_DIR) + path: $(HOME)/miniconda3 cacheHitVar: CONDA_CACHE_RESTORED - bash: | conda env create -q --file ci\\deps\\azure-windows-$(CONDA_PY).yaml diff --git a/ci/setup_conda.sh b/ci/setup_conda.sh new file mode 100755 index 0000000000000..0f10ea8f7d74b --- /dev/null +++ b/ci/setup_conda.sh @@ -0,0 +1,61 @@ +#!/bin/bash -e + +# edit the locale file if needed +if [[ "$(uname)" == "Linux" && -n "$LC_ALL" ]]; then + echo "Adding locale to the first line of pandas/__init__.py" + rm -f pandas/__init__.pyc + SEDC="3iimport locale\nlocale.setlocale(locale.LC_ALL, '$LC_ALL')\n" + sed -i "$SEDC" pandas/__init__.py + + echo "[head -4 pandas/__init__.py]" + head -4 pandas/__init__.py + echo +fi + +echo "Install Miniconda" +DEFAULT_CONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest" +if [[ "$(uname -m)" == 'aarch64' ]]; then + CONDA_URL="https://github.com/conda-forge/miniforge/releases/download/4.10.1-4/Miniforge3-4.10.1-4-Linux-aarch64.sh" +elif [[ "$(uname)" == 'Linux' ]]; then + if [[ "$BITS32" == "yes" ]]; then + CONDA_URL="$DEFAULT_CONDA_URL-Linux-x86.sh" + else + CONDA_URL="$DEFAULT_CONDA_URL-Linux-x86_64.sh" + fi +elif [[ "$(uname)" == 'Darwin' ]]; then + CONDA_URL="$DEFAULT_CONDA_URL-MacOSX-x86_64.sh" +else + echo "OS $(uname) not supported" + exit 1 +fi +echo "Downloading $CONDA_URL" +wget -q $CONDA_URL -O miniconda.sh +chmod +x miniconda.sh + +MINICONDA_DIR="$HOME/miniconda3" +rm -rf $MINICONDA_DIR +./miniconda.sh -b -p $MINICONDA_DIR +export PATH=$MINICONDA_DIR/bin:$PATH + +echo +echo "which conda" +which conda + +echo +echo "update conda" +conda config --set ssl_verify false +conda config --set quiet true --set always_yes true --set changeps1 false +conda install pip conda # create conda to create a historical artifact for pip & setuptools +conda update -n base conda + +echo "conda info -a" +conda info -a + +echo "source deactivate" +source deactivate + +echo "conda list (root environment)" +conda list + +# Clean up any left-over from a previous build +conda remove --all -q -y -n pandas-dev diff --git a/ci/setup_env.sh b/ci/setup_env.sh index a6759797b5bb4..27d730d7b4b14 100755 --- a/ci/setup_env.sh +++ b/ci/setup_env.sh @@ -1,71 +1,9 @@ #!/bin/bash -e -# edit the locale file if needed -if [[ "$(uname)" == "Linux" && -n "$LC_ALL" ]]; then - echo "Adding locale to the first line of pandas/__init__.py" - rm -f pandas/__init__.pyc - SEDC="3iimport locale\nlocale.setlocale(locale.LC_ALL, '$LC_ALL')\n" - sed -i "$SEDC" pandas/__init__.py - - echo "[head -4 pandas/__init__.py]" - head -4 pandas/__init__.py - echo -fi - - -echo "Install Miniconda" -DEFAULT_CONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest" -if [[ "$(uname -m)" == 'aarch64' ]]; then - CONDA_URL="https://github.com/conda-forge/miniforge/releases/download/4.10.1-4/Miniforge3-4.10.1-4-Linux-aarch64.sh" -elif [[ "$(uname)" == 'Linux' ]]; then - if [[ "$BITS32" == "yes" ]]; then - CONDA_URL="$DEFAULT_CONDA_URL-Linux-x86.sh" - else - CONDA_URL="$DEFAULT_CONDA_URL-Linux-x86_64.sh" - fi -elif [[ "$(uname)" == 'Darwin' ]]; then - CONDA_URL="$DEFAULT_CONDA_URL-MacOSX-x86_64.sh" -else - echo "OS $(uname) not supported" - exit 1 -fi -echo "Downloading $CONDA_URL" -wget -q $CONDA_URL -O miniconda.sh -chmod +x miniconda.sh - -MINICONDA_DIR="$HOME/miniconda3" -rm -rf $MINICONDA_DIR -./miniconda.sh -b -p $MINICONDA_DIR -export PATH=$MINICONDA_DIR/bin:$PATH - -echo -echo "which conda" -which conda - -echo -echo "update conda" -conda config --set ssl_verify false -conda config --set quiet true --set always_yes true --set changeps1 false -conda install pip conda # create conda to create a historical artifact for pip & setuptools -conda update -n base conda - -echo "conda info -a" -conda info -a - -echo "source deactivate" -source deactivate - -echo "conda list (root environment)" -conda list - -# Clean up any left-over from a previous build -conda remove --all -q -y -n pandas-dev - echo echo "conda env create -q --file=${ENV_FILE}" time conda env create -q --file="${ENV_FILE}" - if [[ "$BITS32" == "yes" ]]; then # activate 32-bit compiler export CONDA_BUILD=1 @@ -118,10 +56,10 @@ conda list # Install DB for Linux if [[ -n ${SQL:0} ]]; then - echo "installing dbs" - mysql -e 'create database pandas_nosetest;' - psql -c 'create database pandas_nosetest;' -U postgres + echo "installing dbs" + mysql -e 'create database pandas_nosetest;' + psql -c 'create database pandas_nosetest;' -U postgres else - echo "not using dbs on non-linux Travis builds or Azure Pipelines" + echo "not using dbs on non-linux Travis builds or Azure Pipelines" fi echo "done" From 9de1b58b98adbbb0dfb0f66caf1e2250a634ec00 Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Mon, 25 Oct 2021 16:27:51 +0100 Subject: [PATCH 05/27] ENH: Should be a path not a string --- ci/azure/posix.yml | 2 +- ci/azure/windows.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index caed8b1717c92..1b8803dce9d1c 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -27,7 +27,7 @@ jobs: - task: Cache@2 displayName: Use cached Anaconda environment inputs: - key: 'conda | "$(Agent.OS)" | "$(ENV_FILE)"' + key: 'conda | "$(Agent.OS)" | $(ENV_FILE)' path: $(HOME)/miniconda3 restoreKeys: | python | "$(Agent.OS)" diff --git a/ci/azure/windows.yml b/ci/azure/windows.yml index 34a0f4dc1c005..2268046003614 100644 --- a/ci/azure/windows.yml +++ b/ci/azure/windows.yml @@ -47,7 +47,7 @@ jobs: - task: Cache@2 displayName: Use cached Anaconda environment inputs: - key: 'conda | "$(Agent.OS)" | "$(ENV_FILE)"' + key: 'conda | "$(Agent.OS)" | $(ENV_FILE)' restoreKeys: | python | "$(Agent.OS)" python From 8950f4216793c5472486e118ae953f4a2ecfbeea Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Mon, 25 Oct 2021 16:43:59 +0100 Subject: [PATCH 06/27] ENH: Move pandas build - should be its own task --- ci/azure/posix.yml | 12 +++++++++++- ci/setup_env.sh | 15 --------------- 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 1b8803dce9d1c..53a7d54b3b9b0 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -38,9 +38,19 @@ jobs: displayName: 'Install Miniconda in CI env' - script: ci/setup_env.sh - displayName: 'Install conda dependencies and build pandas' + displayName: 'Install conda dependencies in env' condition: eq(variables.CONDA_CACHE_RESTORED, 'false') + - script: | + source activate pandas-dev + conda list pandas + python setup.py build_ext -q -j2 + python -m pip install --no-deps -U pip wheel setuptools + python -m pip install --no-build-isolation -e . + echo "conda list" + conda list + displayName: 'Build extensions and install pandas' + - script: | source activate pandas-dev ci/run_tests.sh diff --git a/ci/setup_env.sh b/ci/setup_env.sh index 27d730d7b4b14..9bb963a8ce1cb 100755 --- a/ci/setup_env.sh +++ b/ci/setup_env.sh @@ -38,21 +38,6 @@ echo echo "conda list pandas" conda list pandas -# Make sure any error below is reported as such - -echo "[Build extensions]" -python setup.py build_ext -q -j2 - -echo "[Updating pip]" -python -m pip install --no-deps -U pip wheel setuptools - -echo "[Install pandas]" -python -m pip install --no-build-isolation -e . - -echo -echo "conda list" -conda list - # Install DB for Linux if [[ -n ${SQL:0} ]]; then From 73711505f4b9384e634882d33830bc4ab9c02e73 Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Mon, 25 Oct 2021 17:26:44 +0100 Subject: [PATCH 07/27] ENH: Ensure the cache path exists - Windows --- azure-pipelines.yml | 1 + ci/azure/windows.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 6c685d09ab55a..4ea05fe8fd637 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -18,6 +18,7 @@ pr: variables: PYTEST_WORKERS: auto PYTEST_TARGET: pandas + CONDA_CACHE_DIR: $(Pipeline.Workspace)/.condarc jobs: # Mac and Linux use the same template diff --git a/ci/azure/windows.yml b/ci/azure/windows.yml index 2268046003614..eba26fbe5e1f3 100644 --- a/ci/azure/windows.yml +++ b/ci/azure/windows.yml @@ -51,7 +51,7 @@ jobs: restoreKeys: | python | "$(Agent.OS)" python - path: $(HOME)/miniconda3 + path: $CONDA_CACHE_DIR cacheHitVar: CONDA_CACHE_RESTORED - bash: | conda env create -q --file ci\\deps\\azure-windows-$(CONDA_PY).yaml From 8d93bc8ebe180bdae77885a3920631a135f5c4bf Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Mon, 25 Oct 2021 18:09:23 +0100 Subject: [PATCH 08/27] ENH: cache miniconda too --- ci/azure/posix.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 53a7d54b3b9b0..a49546cfdb784 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -28,7 +28,7 @@ jobs: displayName: Use cached Anaconda environment inputs: key: 'conda | "$(Agent.OS)" | $(ENV_FILE)' - path: $(HOME)/miniconda3 + path: $(HOME)/miniconda3/ restoreKeys: | python | "$(Agent.OS)" python @@ -36,6 +36,7 @@ jobs: - script: ci/setup_conda.sh displayName: 'Install Miniconda in CI env' + condition: eq(variables.CONDA_CACHE_RESTORED, 'false') - script: ci/setup_env.sh displayName: 'Install conda dependencies in env' From 58e8df54673adb0631994862c81cf67f5d98485f Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Mon, 25 Oct 2021 18:21:53 +0100 Subject: [PATCH 09/27] ENH: Ensure cache dir exists - Windows --- ci/azure/windows.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/azure/windows.yml b/ci/azure/windows.yml index eba26fbe5e1f3..ecf5cc14c6a5b 100644 --- a/ci/azure/windows.yml +++ b/ci/azure/windows.yml @@ -58,6 +58,7 @@ jobs: displayName: 'Create anaconda environment' condition: eq(variables.CONDA_CACHE_RESTORED, 'false') - bash: | + mkdir -p $CONDA_CACHE_DIR source activate pandas-dev conda list python setup.py build_ext -q -j 4 From 027684a10eccc7ca61a2898e8d360737585c7576 Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Mon, 25 Oct 2021 20:22:50 +0100 Subject: [PATCH 10/27] ENH: Force make cache dir --- ci/azure/windows.yml | 9 ++++--- ci/setup_conda.sh | 61 -------------------------------------------- 2 files changed, 5 insertions(+), 65 deletions(-) delete mode 100755 ci/setup_conda.sh diff --git a/ci/azure/windows.yml b/ci/azure/windows.yml index ecf5cc14c6a5b..3112efda12e94 100644 --- a/ci/azure/windows.yml +++ b/ci/azure/windows.yml @@ -41,7 +41,9 @@ jobs: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts" Write-Host "##vso[task.prependpath]$HOME/miniconda3/bin" displayName: 'Add conda to PATH' - - script: conda update -q -n base conda + - script: | + conda update -q -n base conda + mkdir -p $(Pipeline.Workspace)/.condarc displayName: 'Update conda' - task: Cache@2 @@ -51,19 +53,18 @@ jobs: restoreKeys: | python | "$(Agent.OS)" python - path: $CONDA_CACHE_DIR + path: $(Pipeline.Workspace)/.condarc cacheHitVar: CONDA_CACHE_RESTORED - bash: | conda env create -q --file ci\\deps\\azure-windows-$(CONDA_PY).yaml displayName: 'Create anaconda environment' condition: eq(variables.CONDA_CACHE_RESTORED, 'false') - bash: | - mkdir -p $CONDA_CACHE_DIR source activate pandas-dev conda list python setup.py build_ext -q -j 4 python -m pip install --no-build-isolation -e . - displayName: 'Build' + displayName: 'Build pandas' - bash: | source activate pandas-dev wmic.exe cpu get caption, deviceid, name, numberofcores, maxclockspeed diff --git a/ci/setup_conda.sh b/ci/setup_conda.sh deleted file mode 100755 index 0f10ea8f7d74b..0000000000000 --- a/ci/setup_conda.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/bash -e - -# edit the locale file if needed -if [[ "$(uname)" == "Linux" && -n "$LC_ALL" ]]; then - echo "Adding locale to the first line of pandas/__init__.py" - rm -f pandas/__init__.pyc - SEDC="3iimport locale\nlocale.setlocale(locale.LC_ALL, '$LC_ALL')\n" - sed -i "$SEDC" pandas/__init__.py - - echo "[head -4 pandas/__init__.py]" - head -4 pandas/__init__.py - echo -fi - -echo "Install Miniconda" -DEFAULT_CONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest" -if [[ "$(uname -m)" == 'aarch64' ]]; then - CONDA_URL="https://github.com/conda-forge/miniforge/releases/download/4.10.1-4/Miniforge3-4.10.1-4-Linux-aarch64.sh" -elif [[ "$(uname)" == 'Linux' ]]; then - if [[ "$BITS32" == "yes" ]]; then - CONDA_URL="$DEFAULT_CONDA_URL-Linux-x86.sh" - else - CONDA_URL="$DEFAULT_CONDA_URL-Linux-x86_64.sh" - fi -elif [[ "$(uname)" == 'Darwin' ]]; then - CONDA_URL="$DEFAULT_CONDA_URL-MacOSX-x86_64.sh" -else - echo "OS $(uname) not supported" - exit 1 -fi -echo "Downloading $CONDA_URL" -wget -q $CONDA_URL -O miniconda.sh -chmod +x miniconda.sh - -MINICONDA_DIR="$HOME/miniconda3" -rm -rf $MINICONDA_DIR -./miniconda.sh -b -p $MINICONDA_DIR -export PATH=$MINICONDA_DIR/bin:$PATH - -echo -echo "which conda" -which conda - -echo -echo "update conda" -conda config --set ssl_verify false -conda config --set quiet true --set always_yes true --set changeps1 false -conda install pip conda # create conda to create a historical artifact for pip & setuptools -conda update -n base conda - -echo "conda info -a" -conda info -a - -echo "source deactivate" -source deactivate - -echo "conda list (root environment)" -conda list - -# Clean up any left-over from a previous build -conda remove --all -q -y -n pandas-dev From 834f8242777a4fa48c10bdd6ccacefdffd2e1c16 Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Mon, 25 Oct 2021 20:24:51 +0100 Subject: [PATCH 11/27] ENH: Revert to original setup_env.sh --- ci/azure/posix.yml | 6 +---- ci/setup_env.sh | 60 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 5 deletions(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index a49546cfdb784..c11728f04dfb6 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -34,12 +34,8 @@ jobs: python cacheHitVar: CONDA_CACHE_RESTORED - - script: ci/setup_conda.sh - displayName: 'Install Miniconda in CI env' - condition: eq(variables.CONDA_CACHE_RESTORED, 'false') - - script: ci/setup_env.sh - displayName: 'Install conda dependencies in env' + displayName: 'Setup environment' condition: eq(variables.CONDA_CACHE_RESTORED, 'false') - script: | diff --git a/ci/setup_env.sh b/ci/setup_env.sh index 9bb963a8ce1cb..2ae117f2e23ce 100755 --- a/ci/setup_env.sh +++ b/ci/setup_env.sh @@ -1,5 +1,65 @@ #!/bin/bash -e +# edit the locale file if needed +if [[ "$(uname)" == "Linux" && -n "$LC_ALL" ]]; then + echo "Adding locale to the first line of pandas/__init__.py" + rm -f pandas/__init__.pyc + SEDC="3iimport locale\nlocale.setlocale(locale.LC_ALL, '$LC_ALL')\n" + sed -i "$SEDC" pandas/__init__.py + + echo "[head -4 pandas/__init__.py]" + head -4 pandas/__init__.py + echo +fi + +echo "Install Miniconda" +DEFAULT_CONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest" +if [[ "$(uname -m)" == 'aarch64' ]]; then + CONDA_URL="https://github.com/conda-forge/miniforge/releases/download/4.10.1-4/Miniforge3-4.10.1-4-Linux-aarch64.sh" +elif [[ "$(uname)" == 'Linux' ]]; then + if [[ "$BITS32" == "yes" ]]; then + CONDA_URL="$DEFAULT_CONDA_URL-Linux-x86.sh" + else + CONDA_URL="$DEFAULT_CONDA_URL-Linux-x86_64.sh" + fi +elif [[ "$(uname)" == 'Darwin' ]]; then + CONDA_URL="$DEFAULT_CONDA_URL-MacOSX-x86_64.sh" +else + echo "OS $(uname) not supported" + exit 1 +fi +echo "Downloading $CONDA_URL" +wget -q $CONDA_URL -O miniconda.sh +chmod +x miniconda.sh + +MINICONDA_DIR="$HOME/miniconda3" +rm -rf $MINICONDA_DIR +./miniconda.sh -b -p $MINICONDA_DIR +export PATH=$MINICONDA_DIR/bin:$PATH + +echo +echo "which conda" +which conda + +echo +echo "update conda" +conda config --set ssl_verify false +conda config --set quiet true --set always_yes true --set changeps1 false +conda install pip conda # create conda to create a historical artifact for pip & setuptools +conda update -n base conda + +echo "conda info -a" +conda info -a + +echo "source deactivate" +source deactivate + +echo "conda list (root environment)" +conda list + +# Clean up any left-over from a previous build +conda remove --all -q -y -n pandas-dev + echo echo "conda env create -q --file=${ENV_FILE}" time conda env create -q --file="${ENV_FILE}" From df307fada387f8e013cae7f57ca1f2b527db559e Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Mon, 25 Oct 2021 20:31:35 +0100 Subject: [PATCH 12/27] ENH: use bash for conda update --- ci/azure/windows.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/azure/windows.yml b/ci/azure/windows.yml index 3112efda12e94..9e938b33d173e 100644 --- a/ci/azure/windows.yml +++ b/ci/azure/windows.yml @@ -41,9 +41,9 @@ jobs: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts" Write-Host "##vso[task.prependpath]$HOME/miniconda3/bin" displayName: 'Add conda to PATH' - - script: | + - bash: | conda update -q -n base conda - mkdir -p $(Pipeline.Workspace)/.condarc + mkdir -p $(CONDA_CACHE_DIR) displayName: 'Update conda' - task: Cache@2 @@ -53,7 +53,7 @@ jobs: restoreKeys: | python | "$(Agent.OS)" python - path: $(Pipeline.Workspace)/.condarc + path: $(CONDA_CACHE_DIR) cacheHitVar: CONDA_CACHE_RESTORED - bash: | conda env create -q --file ci\\deps\\azure-windows-$(CONDA_PY).yaml From 76ba935b66560a8a769f2b24f6fe5f78c3bed317 Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Tue, 26 Oct 2021 11:07:20 +0100 Subject: [PATCH 13/27] ENH: Use Miniconda dir in Windows --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 4ea05fe8fd637..ad71dff35ef5f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -18,7 +18,7 @@ pr: variables: PYTEST_WORKERS: auto PYTEST_TARGET: pandas - CONDA_CACHE_DIR: $(Pipeline.Workspace)/.condarc + CONDA_CACHE_DIR: $(Pipeline.Workspace)/Miniconda/ jobs: # Mac and Linux use the same template From f13efbe8dabe9f289edc1e9f9a92bd3b8d663c75 Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Thu, 28 Oct 2021 16:46:18 +0100 Subject: [PATCH 14/27] Trivial change to trigger pipeline --- ci/azure/windows.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/azure/windows.yml b/ci/azure/windows.yml index 9e938b33d173e..d12858938b604 100644 --- a/ci/azure/windows.yml +++ b/ci/azure/windows.yml @@ -41,6 +41,7 @@ jobs: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts" Write-Host "##vso[task.prependpath]$HOME/miniconda3/bin" displayName: 'Add conda to PATH' + - bash: | conda update -q -n base conda mkdir -p $(CONDA_CACHE_DIR) From 77cd4154df91f119accffd234c9e8efd5b118851 Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Thu, 28 Oct 2021 16:52:48 +0100 Subject: [PATCH 15/27] Revert to base CI --- ci/azure/windows.yml | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/ci/azure/windows.yml b/ci/azure/windows.yml index d12858938b604..ac724a8556b86 100644 --- a/ci/azure/windows.yml +++ b/ci/azure/windows.yml @@ -42,30 +42,18 @@ jobs: Write-Host "##vso[task.prependpath]$HOME/miniconda3/bin" displayName: 'Add conda to PATH' - - bash: | - conda update -q -n base conda - mkdir -p $(CONDA_CACHE_DIR) + - script: conda update -q -n base conda displayName: 'Update conda' - - task: Cache@2 - displayName: Use cached Anaconda environment - inputs: - key: 'conda | "$(Agent.OS)" | $(ENV_FILE)' - restoreKeys: | - python | "$(Agent.OS)" - python - path: $(CONDA_CACHE_DIR) - cacheHitVar: CONDA_CACHE_RESTORED - bash: | conda env create -q --file ci\\deps\\azure-windows-$(CONDA_PY).yaml displayName: 'Create anaconda environment' - condition: eq(variables.CONDA_CACHE_RESTORED, 'false') - bash: | source activate pandas-dev conda list python setup.py build_ext -q -j 4 python -m pip install --no-build-isolation -e . - displayName: 'Build pandas' + displayName: 'Build' - bash: | source activate pandas-dev wmic.exe cpu get caption, deviceid, name, numberofcores, maxclockspeed From 56dc350715cc20e156a2e076e58e1ff436897fef Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Thu, 28 Oct 2021 17:00:08 +0100 Subject: [PATCH 16/27] ENH: Add cache again --- ci/azure/windows.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/ci/azure/windows.yml b/ci/azure/windows.yml index ac724a8556b86..68092b9cb0c9e 100644 --- a/ci/azure/windows.yml +++ b/ci/azure/windows.yml @@ -45,9 +45,20 @@ jobs: - script: conda update -q -n base conda displayName: 'Update conda' + - task: Cache@2 + displayName: Use cached Anaconda environment + inputs: + key: 'conda | "$(Agent.OS)" | $(ENV_FILE)' + path: $(Pipeline.Workspace)/Miniconda/ + restoreKeys: | + python | "$(Agent.OS)" + python + cacheHitVar: CONDA_CACHE_RESTORED + - bash: | conda env create -q --file ci\\deps\\azure-windows-$(CONDA_PY).yaml displayName: 'Create anaconda environment' + - bash: | source activate pandas-dev conda list From 248055652bc1bba9bd258f0278961c31586bcbad Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Thu, 28 Oct 2021 17:01:46 +0100 Subject: [PATCH 17/27] :construction_worker: Add cache hit --- ci/azure/windows.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/azure/windows.yml b/ci/azure/windows.yml index 68092b9cb0c9e..12fd96aedf0a0 100644 --- a/ci/azure/windows.yml +++ b/ci/azure/windows.yml @@ -58,13 +58,14 @@ jobs: - bash: | conda env create -q --file ci\\deps\\azure-windows-$(CONDA_PY).yaml displayName: 'Create anaconda environment' + condition: eq(variables.CONDA_CACHE_RESTORED, 'false') - bash: | source activate pandas-dev conda list python setup.py build_ext -q -j 4 python -m pip install --no-build-isolation -e . - displayName: 'Build' + displayName: 'Build and install pandas' - bash: | source activate pandas-dev wmic.exe cpu get caption, deviceid, name, numberofcores, maxclockspeed From 9a9f1db8dd7a362cac42477ba520277456048a8e Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Thu, 28 Oct 2021 17:34:29 +0100 Subject: [PATCH 18/27] :bug: Fix Cache path --- azure-pipelines.yml | 1 - ci/azure/windows.yml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ad71dff35ef5f..6c685d09ab55a 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -18,7 +18,6 @@ pr: variables: PYTEST_WORKERS: auto PYTEST_TARGET: pandas - CONDA_CACHE_DIR: $(Pipeline.Workspace)/Miniconda/ jobs: # Mac and Linux use the same template diff --git a/ci/azure/windows.yml b/ci/azure/windows.yml index 12fd96aedf0a0..86b4c50944cae 100644 --- a/ci/azure/windows.yml +++ b/ci/azure/windows.yml @@ -49,7 +49,7 @@ jobs: displayName: Use cached Anaconda environment inputs: key: 'conda | "$(Agent.OS)" | $(ENV_FILE)' - path: $(Pipeline.Workspace)/Miniconda/ + path: C:\Miniconda restoreKeys: | python | "$(Agent.OS)" python From 8325e03566fa4c43330ef03cf24b512ee24e5c09 Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Sat, 30 Oct 2021 11:17:03 +0100 Subject: [PATCH 19/27] Trivial change to trigger pipelines --- ci/azure/windows.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/azure/windows.yml b/ci/azure/windows.yml index 86b4c50944cae..0c6c235522c94 100644 --- a/ci/azure/windows.yml +++ b/ci/azure/windows.yml @@ -66,6 +66,7 @@ jobs: python setup.py build_ext -q -j 4 python -m pip install --no-build-isolation -e . displayName: 'Build and install pandas' + - bash: | source activate pandas-dev wmic.exe cpu get caption, deviceid, name, numberofcores, maxclockspeed From 464dddf209ad242c2f291b1de53a8a59406567ea Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Sat, 30 Oct 2021 11:32:33 +0100 Subject: [PATCH 20/27] Print conda info --- ci/azure/windows.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci/azure/windows.yml b/ci/azure/windows.yml index 0c6c235522c94..2e0c9ee2fc44c 100644 --- a/ci/azure/windows.yml +++ b/ci/azure/windows.yml @@ -45,6 +45,9 @@ jobs: - script: conda update -q -n base conda displayName: 'Update conda' + - script: conda info + displayName: 'Display conda info' + - task: Cache@2 displayName: Use cached Anaconda environment inputs: From 1277c6a2d7729a7b54443197d59ede3fc4b35cef Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Sat, 30 Oct 2021 11:38:58 +0100 Subject: [PATCH 21/27] Add conda update to posix --- ci/azure/posix.yml | 4 ++++ ci/azure/windows.yml | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index c11728f04dfb6..7146fa4e95941 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -38,6 +38,10 @@ jobs: displayName: 'Setup environment' condition: eq(variables.CONDA_CACHE_RESTORED, 'false') + - script: | + conda update -q -y -n pandas-dev + disply + - script: | source activate pandas-dev conda list pandas diff --git a/ci/azure/windows.yml b/ci/azure/windows.yml index 2e0c9ee2fc44c..a729e6cd24f17 100644 --- a/ci/azure/windows.yml +++ b/ci/azure/windows.yml @@ -45,14 +45,14 @@ jobs: - script: conda update -q -n base conda displayName: 'Update conda' - - script: conda info + - bash: echo "$(conda info)" displayName: 'Display conda info' - task: Cache@2 displayName: Use cached Anaconda environment inputs: key: 'conda | "$(Agent.OS)" | $(ENV_FILE)' - path: C:\Miniconda + path: $(Pipeline.Workspace)/Miniconda restoreKeys: | python | "$(Agent.OS)" python From 578ec73e15b0ae5c94db374f5b5b06bd42917f83 Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Sat, 30 Oct 2021 11:42:43 +0100 Subject: [PATCH 22/27] :bug: Fix imcomplte script --- ci/azure/posix.yml | 2 +- ci/azure/windows.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 7146fa4e95941..c188d5faacfef 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -40,7 +40,7 @@ jobs: - script: | conda update -q -y -n pandas-dev - disply + displayName: 'Update packages in pandas-dev' - script: | source activate pandas-dev diff --git a/ci/azure/windows.yml b/ci/azure/windows.yml index a729e6cd24f17..66d9fe244b4ac 100644 --- a/ci/azure/windows.yml +++ b/ci/azure/windows.yml @@ -52,7 +52,7 @@ jobs: displayName: Use cached Anaconda environment inputs: key: 'conda | "$(Agent.OS)" | $(ENV_FILE)' - path: $(Pipeline.Workspace)/Miniconda + path: $(Pipeline.Workspace)\.condarc restoreKeys: | python | "$(Agent.OS)" python From 03692a292b8c09889578acd706bdaa05108454dd Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Sat, 30 Oct 2021 11:52:55 +0100 Subject: [PATCH 23/27] Fix update command --- ci/azure/posix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index c188d5faacfef..893084d1850c4 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -39,7 +39,7 @@ jobs: condition: eq(variables.CONDA_CACHE_RESTORED, 'false') - script: | - conda update -q -y -n pandas-dev + conda update -q -y --all -n pandas-dev displayName: 'Update packages in pandas-dev' - script: | From a26a8963dc7dca71aaee6dd6f5e09fd0a93314ee Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Mon, 1 Nov 2021 17:48:24 +0000 Subject: [PATCH 24/27] Update ci/azure/posix.yml Co-authored-by: Thomas Li <47963215+lithomas1@users.noreply.github.com> --- ci/azure/posix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 893084d1850c4..1a6a9fa226359 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -39,7 +39,7 @@ jobs: condition: eq(variables.CONDA_CACHE_RESTORED, 'false') - script: | - conda update -q -y --all -n pandas-dev + conda update -q -y -n pandas-dev --file $ENV_FILE displayName: 'Update packages in pandas-dev' - script: | From 4df7ede54adfbd61839b3b2f7ecbe2d4855ba190 Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Mon, 1 Nov 2021 17:50:25 +0000 Subject: [PATCH 25/27] Revert Windows ci --- ci/azure/windows.yml | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/ci/azure/windows.yml b/ci/azure/windows.yml index 66d9fe244b4ac..3bd20b1399be2 100644 --- a/ci/azure/windows.yml +++ b/ci/azure/windows.yml @@ -41,35 +41,18 @@ jobs: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts" Write-Host "##vso[task.prependpath]$HOME/miniconda3/bin" displayName: 'Add conda to PATH' - - script: conda update -q -n base conda displayName: 'Update conda' - - bash: echo "$(conda info)" - displayName: 'Display conda info' - - - task: Cache@2 - displayName: Use cached Anaconda environment - inputs: - key: 'conda | "$(Agent.OS)" | $(ENV_FILE)' - path: $(Pipeline.Workspace)\.condarc - restoreKeys: | - python | "$(Agent.OS)" - python - cacheHitVar: CONDA_CACHE_RESTORED - - bash: | conda env create -q --file ci\\deps\\azure-windows-$(CONDA_PY).yaml displayName: 'Create anaconda environment' - condition: eq(variables.CONDA_CACHE_RESTORED, 'false') - - bash: | source activate pandas-dev conda list python setup.py build_ext -q -j 4 python -m pip install --no-build-isolation -e . - displayName: 'Build and install pandas' - + displayName: 'Build' - bash: | source activate pandas-dev wmic.exe cpu get caption, deviceid, name, numberofcores, maxclockspeed From dfcd0d9e7591ac312f8c9a9a0a957ca79f5155e3 Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Tue, 18 Jan 2022 13:20:13 +0000 Subject: [PATCH 26/27] Fix env update --- ci/azure/posix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index cf317de217f5a..87fcbe99de438 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -58,7 +58,7 @@ jobs: condition: eq(variables.CONDA_CACHE_RESTORED, 'false') - script: | - conda update -q -y -n pandas-dev --file $ENV_FILE + conda env update -q -y -n pandas-dev --file $ENV_FILE displayName: 'Update packages in pandas-dev' - script: | From bff1bee19e32c276965555ffe5f8909409a49946 Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Tue, 18 Jan 2022 14:51:46 +0000 Subject: [PATCH 27/27] Remove illegal flag --- ci/azure/posix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 87fcbe99de438..8bdafc85b2c2a 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -58,7 +58,7 @@ jobs: condition: eq(variables.CONDA_CACHE_RESTORED, 'false') - script: | - conda env update -q -y -n pandas-dev --file $ENV_FILE + conda env update -q -n pandas-dev --file $ENV_FILE displayName: 'Update packages in pandas-dev' - script: |