diff --git a/.gitpod.yml b/.gitpod.yml
index dd58c4df72965..25b0ac103bc34 100644
--- a/.gitpod.yml
+++ b/.gitpod.yml
@@ -14,7 +14,6 @@ tasks:
       git pull --unshallow  # need to force this else the prebuild fails
       git fetch --tags
       python setup.py build_ext -j 4
-      python -m pip install -e . --no-build-isolation
       echo "🛠 Completed rebuilding Pandas!! 🛠 "
       echo "✨ Pre-build complete! You can close this terminal ✨ "
 
diff --git a/gitpod/gitpod.Dockerfile b/gitpod/gitpod.Dockerfile
index 77268577e502b..ab3f25b231e67 100644
--- a/gitpod/gitpod.Dockerfile
+++ b/gitpod/gitpod.Dockerfile
@@ -34,6 +34,7 @@ WORKDIR ${WORKSPACE}
 # Build pandas to populate the cache used by ccache
 RUN git config --global --add safe.directory /workspace/pandas
 RUN conda activate ${CONDA_ENV} && \
+    python -m pip install -e . --no-build-isolation && \
     python setup.py build_ext --inplace && \
     ccache -s
 
@@ -43,4 +44,5 @@ RUN rm -rf ${WORKSPACE}
 
 # -----------------------------------------------------------------------------
 # Always return to non privileged user
+RUN chown -R gitpod:gitpod /home/gitpod/.cache/
 USER gitpod