diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml
index 02a4a9ad44865..8bdafc85b2c2a 100644
--- a/ci/azure/posix.yml
+++ b/ci/azure/posix.yml
@@ -43,8 +43,33 @@ jobs:
     - script: echo '##vso[task.prependpath]$(HOME)/miniconda3/bin'
       displayName: 'Set conda path'
 
+    - task: Cache@2
+      displayName: Use cached Anaconda environment
+      inputs:
+        key: 'conda | "$(Agent.OS)" | $(ENV_FILE)'
+        path: $(HOME)/miniconda3/
+        restoreKeys: |
+          python | "$(Agent.OS)"
+          python
+        cacheHitVar: CONDA_CACHE_RESTORED
+
     - script: ci/setup_env.sh
-      displayName: 'Setup environment and build pandas'
+      displayName: 'Setup environment'
+      condition: eq(variables.CONDA_CACHE_RESTORED, 'false')
+
+    - script: |
+        conda env update -q -n pandas-dev --file $ENV_FILE
+      displayName: 'Update packages in pandas-dev'
+
+    - script: |
+        source activate pandas-dev
+        conda list pandas
+        python setup.py build_ext -q -j2
+        python -m pip install --no-deps -U pip wheel setuptools
+        python -m pip install --no-build-isolation -e .
+        echo "conda list"
+        conda list
+      displayName: 'Build extensions and install pandas'
 
     - script: |
         source activate pandas-dev
diff --git a/ci/setup_env.sh b/ci/setup_env.sh
index d51ff98b241a6..1ef994cf62f7c 100755
--- a/ci/setup_env.sh
+++ b/ci/setup_env.sh
@@ -12,7 +12,6 @@ if [[ "$(uname)" == "Linux" && -n "$LC_ALL" ]]; then
     echo
 fi
 
-
 echo "Install Miniconda"
 DEFAULT_CONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest"
 if [[ "$(uname -m)" == 'aarch64' ]]; then
@@ -26,8 +25,8 @@ elif [[ "$(uname)" == 'Linux' ]]; then
 elif [[ "$(uname)" == 'Darwin' ]]; then
     CONDA_URL="$DEFAULT_CONDA_URL-MacOSX-x86_64.sh"
 else
-  echo "OS $(uname) not supported"
-  exit 1
+    echo "OS $(uname) not supported"
+    exit 1
 fi
 echo "Downloading $CONDA_URL"
 wget -q $CONDA_URL -O miniconda.sh
@@ -46,7 +45,7 @@ echo
 echo "update conda"
 conda config --set ssl_verify false
 conda config --set quiet true --set always_yes true --set changeps1 false
-conda install pip conda  # create conda to create a historical artifact for pip & setuptools
+conda install pip conda # create conda to create a historical artifact for pip & setuptools
 conda update -n base conda
 conda install -y -c conda-forge mamba
 
@@ -66,7 +65,6 @@ echo
 echo "mamba env create -q --file=${ENV_FILE}"
 time mamba env create -q --file="${ENV_FILE}"
 
-
 if [[ "$BITS32" == "yes" ]]; then
     # activate 32-bit compiler
     export CONDA_BUILD=1