From 5ab51d1f86782ec446b6828bc886a4dc66f8c9f7 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sat, 3 Dec 2022 11:34:48 -0800 Subject: [PATCH 1/8] dockerfiles for hub --- ci/dockerfiles/Dockerfile.alpine | 17 +++++++++++++++++ ci/dockerfiles/Dockerfile.mamba-all | 10 ++++++++++ ci/dockerfiles/Dockerfile.mamba-minimal | 18 ++++++++++++++++++ ci/dockerfiles/Dockerfile.pip-all | 12 ++++++++++++ ci/dockerfiles/Dockerfile.pip-minimal | 16 ++++++++++++++++ 5 files changed, 73 insertions(+) create mode 100644 ci/dockerfiles/Dockerfile.alpine create mode 100644 ci/dockerfiles/Dockerfile.mamba-all create mode 100644 ci/dockerfiles/Dockerfile.mamba-minimal create mode 100644 ci/dockerfiles/Dockerfile.pip-all create mode 100644 ci/dockerfiles/Dockerfile.pip-minimal diff --git a/ci/dockerfiles/Dockerfile.alpine b/ci/dockerfiles/Dockerfile.alpine new file mode 100644 index 0000000000000..b435a817c1f46 --- /dev/null +++ b/ci/dockerfiles/Dockerfile.alpine @@ -0,0 +1,17 @@ +FROM python:3.10.8-alpine + +RUN apk update & apk upgrade +RUN apk add build-base + +RUN python -m pip install \ + cython \ + hypothesis \ + numpy \ + pytest \ + pytest-asyncio \ + pytz \ + python-dateutil \ + versioneer + +WORKDIR /home/pandas +CMD ["/bin/sh"] \ No newline at end of file diff --git a/ci/dockerfiles/Dockerfile.mamba-all b/ci/dockerfiles/Dockerfile.mamba-all new file mode 100644 index 0000000000000..d6569c2308961 --- /dev/null +++ b/ci/dockerfiles/Dockerfile.mamba-all @@ -0,0 +1,10 @@ +FROM quay.io/condaforge/mambaforge + +RUN apt update && apt upgrade -y +RUN DEBIAN_FRONTEND=noninteractive apt install -y tzdata + +RUN mamba env create -f \ + https://raw.githubusercontent.com/pandas-dev/pandas/main/environment.yml + +WORKDIR /home/pandas +CMD ["/bin/sh"] \ No newline at end of file diff --git a/ci/dockerfiles/Dockerfile.mamba-minimal b/ci/dockerfiles/Dockerfile.mamba-minimal new file mode 100644 index 0000000000000..601b0410acc66 --- /dev/null +++ b/ci/dockerfiles/Dockerfile.mamba-minimal @@ -0,0 +1,18 @@ +FROM quay.io/condaforge/mambaforge + +RUN apt update && apt upgrade -y +RUN DEBIAN_FRONTEND=noninteractive apt install -y tzdata + +RUN mamba create -n pandas-dev \ + cython \ + hypothesis \ + numpy \ + pytest \ + pytest-asyncio \ + python=3.10.8 \ + pytz \ + python-dateutil \ + versioneer + +WORKDIR /home/pandas +CMD ["/bin/sh"] \ No newline at end of file diff --git a/ci/dockerfiles/Dockerfile.pip-all b/ci/dockerfiles/Dockerfile.pip-all new file mode 100644 index 0000000000000..c801b5ec7d3c2 --- /dev/null +++ b/ci/dockerfiles/Dockerfile.pip-all @@ -0,0 +1,12 @@ +FROM python:3.10.8 + +RUN apt update && apt -y upgrade + +# hdf5 needed for pytables installation +RUN apt-get install -y libhdf5-dev + +RUN python -m pip install --use-deprecated=legacy-resolver \ + -r https://raw.githubusercontent.com/pandas-dev/pandas/main/requirements-dev.txt + +WORKDIR /home/pandas +CMD ["/bin/sh"] diff --git a/ci/dockerfiles/Dockerfile.pip-minimal b/ci/dockerfiles/Dockerfile.pip-minimal new file mode 100644 index 0000000000000..338c0ccf08b04 --- /dev/null +++ b/ci/dockerfiles/Dockerfile.pip-minimal @@ -0,0 +1,16 @@ +FROM python:3.10.8 + +RUN apt update && apt upgrade -y + +RUN python -m pip install \ + cython \ + hypothesis \ + numpy \ + pytest \ + pytest-asyncio \ + pytz \ + python-dateutil \ + versioneer + +WORKDIR /home/pandas +CMD ["/bin/sh"] From e48ced4c6fd3f54e87e38e75cd3a4c62360b5388 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sat, 3 Dec 2022 11:45:48 -0800 Subject: [PATCH 2/8] initial docker images for developers --- ci/dockerfiles/Dockerfile.alpine | 2 +- ci/dockerfiles/Dockerfile.mamba-all | 2 +- ci/dockerfiles/Dockerfile.mamba-minimal | 2 +- ci/dockerfiles/Dockerfile.pip-all | 2 +- ci/dockerfiles/Dockerfile.pip-minimal | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ci/dockerfiles/Dockerfile.alpine b/ci/dockerfiles/Dockerfile.alpine index b435a817c1f46..73b30f1ccce0e 100644 --- a/ci/dockerfiles/Dockerfile.alpine +++ b/ci/dockerfiles/Dockerfile.alpine @@ -14,4 +14,4 @@ RUN python -m pip install \ versioneer WORKDIR /home/pandas -CMD ["/bin/sh"] \ No newline at end of file +CMD ["/bin/sh"] diff --git a/ci/dockerfiles/Dockerfile.mamba-all b/ci/dockerfiles/Dockerfile.mamba-all index d6569c2308961..1523e2a5dc032 100644 --- a/ci/dockerfiles/Dockerfile.mamba-all +++ b/ci/dockerfiles/Dockerfile.mamba-all @@ -7,4 +7,4 @@ RUN mamba env create -f \ https://raw.githubusercontent.com/pandas-dev/pandas/main/environment.yml WORKDIR /home/pandas -CMD ["/bin/sh"] \ No newline at end of file +CMD ["/bin/sh"] diff --git a/ci/dockerfiles/Dockerfile.mamba-minimal b/ci/dockerfiles/Dockerfile.mamba-minimal index 601b0410acc66..beeff9eff7e6d 100644 --- a/ci/dockerfiles/Dockerfile.mamba-minimal +++ b/ci/dockerfiles/Dockerfile.mamba-minimal @@ -15,4 +15,4 @@ RUN mamba create -n pandas-dev \ versioneer WORKDIR /home/pandas -CMD ["/bin/sh"] \ No newline at end of file +CMD ["conda", "run", "-n", "pandas-dev", "/bin/bash", "-c"] diff --git a/ci/dockerfiles/Dockerfile.pip-all b/ci/dockerfiles/Dockerfile.pip-all index c801b5ec7d3c2..d22a02593e4e0 100644 --- a/ci/dockerfiles/Dockerfile.pip-all +++ b/ci/dockerfiles/Dockerfile.pip-all @@ -9,4 +9,4 @@ RUN python -m pip install --use-deprecated=legacy-resolver \ -r https://raw.githubusercontent.com/pandas-dev/pandas/main/requirements-dev.txt WORKDIR /home/pandas -CMD ["/bin/sh"] +CMD ["/bin/bash"] diff --git a/ci/dockerfiles/Dockerfile.pip-minimal b/ci/dockerfiles/Dockerfile.pip-minimal index 338c0ccf08b04..fe5ae618263d8 100644 --- a/ci/dockerfiles/Dockerfile.pip-minimal +++ b/ci/dockerfiles/Dockerfile.pip-minimal @@ -13,4 +13,4 @@ RUN python -m pip install \ versioneer WORKDIR /home/pandas -CMD ["/bin/sh"] +CMD ["/bin/bash"] From 335a884ba57fae815ac4593d5a6fde46e03553ab Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sat, 3 Dec 2022 13:06:35 -0800 Subject: [PATCH 3/8] mamba auto activate fixes --- ci/dockerfiles/Dockerfile.mamba-all | 4 +++- ci/dockerfiles/Dockerfile.mamba-minimal | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/ci/dockerfiles/Dockerfile.mamba-all b/ci/dockerfiles/Dockerfile.mamba-all index 1523e2a5dc032..61c8e8685b56e 100644 --- a/ci/dockerfiles/Dockerfile.mamba-all +++ b/ci/dockerfiles/Dockerfile.mamba-all @@ -6,5 +6,7 @@ RUN DEBIAN_FRONTEND=noninteractive apt install -y tzdata RUN mamba env create -f \ https://raw.githubusercontent.com/pandas-dev/pandas/main/environment.yml +RUN mamba init +RUN echo "\nmamba activate pandas-dev" >> ~/.bashrc + WORKDIR /home/pandas -CMD ["/bin/sh"] diff --git a/ci/dockerfiles/Dockerfile.mamba-minimal b/ci/dockerfiles/Dockerfile.mamba-minimal index beeff9eff7e6d..258c13c73b912 100644 --- a/ci/dockerfiles/Dockerfile.mamba-minimal +++ b/ci/dockerfiles/Dockerfile.mamba-minimal @@ -14,5 +14,6 @@ RUN mamba create -n pandas-dev \ python-dateutil \ versioneer +RUN mamba init +RUN echo "\nmamba activate pandas-dev" >> ~/.bashrc WORKDIR /home/pandas -CMD ["conda", "run", "-n", "pandas-dev", "/bin/bash", "-c"] From 17a56c83971f47fbd51d2256c35123aada02b240 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sat, 3 Dec 2022 13:19:10 -0800 Subject: [PATCH 4/8] whitespace fixup --- ci/dockerfiles/Dockerfile.pip-minimal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/dockerfiles/Dockerfile.pip-minimal b/ci/dockerfiles/Dockerfile.pip-minimal index fe5ae618263d8..353b80e08b07e 100644 --- a/ci/dockerfiles/Dockerfile.pip-minimal +++ b/ci/dockerfiles/Dockerfile.pip-minimal @@ -7,7 +7,7 @@ RUN python -m pip install \ hypothesis \ numpy \ pytest \ - pytest-asyncio \ + pytest-asyncio \ pytz \ python-dateutil \ versioneer From 2044ddfaf98e9edf56eebcc306cd105f70eb1c4c Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Mon, 5 Dec 2022 14:56:02 -0800 Subject: [PATCH 5/8] mamba cleanups and documentation --- Dockerfile | 13 ----- ci/dockerfiles/Dockerfile.mamba-all | 1 + ci/dockerfiles/Dockerfile.mamba-minimal | 2 + .../development/contributing_environment.rst | 52 +++++++++---------- 4 files changed, 29 insertions(+), 39 deletions(-) delete mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index c987461e8cbb8..0000000000000 --- a/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -FROM python:3.10.8 -WORKDIR /home/pandas - -RUN apt-get update && apt-get -y upgrade -RUN apt-get install -y build-essential - -# hdf5 needed for pytables installation -RUN apt-get install -y libhdf5-dev - -RUN python -m pip install --upgrade pip -RUN python -m pip install --use-deprecated=legacy-resolver \ - -r https://raw.githubusercontent.com/pandas-dev/pandas/main/requirements-dev.txt -CMD ["/bin/bash"] diff --git a/ci/dockerfiles/Dockerfile.mamba-all b/ci/dockerfiles/Dockerfile.mamba-all index 61c8e8685b56e..dda31596d2a39 100644 --- a/ci/dockerfiles/Dockerfile.mamba-all +++ b/ci/dockerfiles/Dockerfile.mamba-all @@ -8,5 +8,6 @@ RUN mamba env create -f \ RUN mamba init RUN echo "\nmamba activate pandas-dev" >> ~/.bashrc +RUN mamba clean --all -qy WORKDIR /home/pandas diff --git a/ci/dockerfiles/Dockerfile.mamba-minimal b/ci/dockerfiles/Dockerfile.mamba-minimal index 258c13c73b912..0e49053f536b5 100644 --- a/ci/dockerfiles/Dockerfile.mamba-minimal +++ b/ci/dockerfiles/Dockerfile.mamba-minimal @@ -16,4 +16,6 @@ RUN mamba create -n pandas-dev \ RUN mamba init RUN echo "\nmamba activate pandas-dev" >> ~/.bashrc +RUN mamba clean --all -qy + WORKDIR /home/pandas diff --git a/doc/source/development/contributing_environment.rst b/doc/source/development/contributing_environment.rst index 69f7f054d865d..bd6a11a4e1780 100644 --- a/doc/source/development/contributing_environment.rst +++ b/doc/source/development/contributing_environment.rst @@ -220,41 +220,41 @@ Option 2: creating an environment using Docker ---------------------------------------------- Instead of manually setting up a development environment, you can use `Docker -`_ to automatically create the environment with just several -commands. pandas provides a ``DockerFile`` in the root directory to build a Docker image -with a full pandas development environment. +`_. pandas provides pre-built images that serve a +variety of users. These images include: -**Docker Commands** + * alpine - a lightweight image for the absolute minimalist (note: this is experimental) + * pip-minimal - a pip-based installation with the minimum set of packages for building / testing + * mamba-minimal - a mamba-based installation with the minimum set of packages for building / testing + * pip-all - a pip-based installation with all testing dependencies + * mamba-all - a mamba-based installation with all testing dependencies -Build the Docker image:: +If you are a new user and the image size is no concern to you, we suggest opting for either image +that includes all of the dependencies, as this will ensure you can run the test suite without any +caveats. - # Build the image - docker build -t pandas-dev . +To use any of the images, you should first start with ``docker pull willayd/pandas-dev:``, +where tag is one of *alpine*, *pip-minimal*, *mamba-minimal*, *pip-all* or *mamba-all*. You can then run +the image without any extra configuration. -Run Container:: - - # Run a container and bind your local repo to the container - # This command assumes you are running from your local repo - # but if not alter ${PWD} to match your local repo path - docker run -it --rm -v ${PWD}:/home/pandas pandas-dev - -When inside the running container you can build and install pandas the same way as the other methods +To illustrate, if you wanted to use the *pip-all* image, from the root of your local pandas project +you would run: .. code-block:: bash - python setup.py build_ext -j 4 - python -m pip install -e . --no-build-isolation --no-use-pep517 + docker pull willayd/pandas-dev:pip-all + docker run --rm -it -v ${PWD}:/home/pandas willayd/pandas-dev:pip-all + +Similarly for *mamba-all* -*Even easier, you can integrate Docker with the following IDEs:* +.. code-block:: bash -**Visual Studio Code** + docker pull willayd/pandas-dev:mamba-all + docker run --rm -it -v ${PWD}:/home/pandas willayd/pandas-dev:mamba-all -You can use the DockerFile to launch a remote session with Visual Studio Code, -a popular free IDE, using the ``.devcontainer.json`` file. -See https://code.visualstudio.com/docs/remote/containers for details. +The *mamba-* images will automatically activate the appropriate virtual environment for you on entry. -**PyCharm (Professional)** +.. note:: -Enable Docker support and use the Services tool window to build and manage images as well as -run and interact with containers. -See https://www.jetbrains.com/help/pycharm/docker.html for details. + You may run the images from a directory besides the root of the pandas project - just be + sure to substitute ${PWD} in the commands above to point to your local pandas repository From ec6484c524f5469cfd67a01fbf94f49b908d5ead Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Mon, 5 Dec 2022 17:59:50 -0800 Subject: [PATCH 6/8] more minimalism --- ci/dockerfiles/Dockerfile.alpine | 13 +++---------- ci/dockerfiles/Dockerfile.pip-minimal | 14 ++++---------- ci/dockerfiles/requirements-minimal.txt | 8 ++++++++ 3 files changed, 15 insertions(+), 20 deletions(-) create mode 100644 ci/dockerfiles/requirements-minimal.txt diff --git a/ci/dockerfiles/Dockerfile.alpine b/ci/dockerfiles/Dockerfile.alpine index 73b30f1ccce0e..17c08a1589b20 100644 --- a/ci/dockerfiles/Dockerfile.alpine +++ b/ci/dockerfiles/Dockerfile.alpine @@ -1,17 +1,10 @@ FROM python:3.10.8-alpine RUN apk update & apk upgrade -RUN apk add build-base +RUN apk add gcc g++ libc-dev -RUN python -m pip install \ - cython \ - hypothesis \ - numpy \ - pytest \ - pytest-asyncio \ - pytz \ - python-dateutil \ - versioneer +COPY requirements-minimal.txt /tmp +RUN python -m pip install -r /tmp/requirements-minimal.txt WORKDIR /home/pandas CMD ["/bin/sh"] diff --git a/ci/dockerfiles/Dockerfile.pip-minimal b/ci/dockerfiles/Dockerfile.pip-minimal index 353b80e08b07e..b88ab50b89136 100644 --- a/ci/dockerfiles/Dockerfile.pip-minimal +++ b/ci/dockerfiles/Dockerfile.pip-minimal @@ -1,16 +1,10 @@ -FROM python:3.10.8 +FROM python:3.10.8-slim RUN apt update && apt upgrade -y +RUN apt install -y gcc g++ -RUN python -m pip install \ - cython \ - hypothesis \ - numpy \ - pytest \ - pytest-asyncio \ - pytz \ - python-dateutil \ - versioneer +COPY requirements-minimal.txt /tmp +RUN python -m pip install -r /tmp/requirements-minimal.txt WORKDIR /home/pandas CMD ["/bin/bash"] diff --git a/ci/dockerfiles/requirements-minimal.txt b/ci/dockerfiles/requirements-minimal.txt new file mode 100644 index 0000000000000..7de6f10d2824f --- /dev/null +++ b/ci/dockerfiles/requirements-minimal.txt @@ -0,0 +1,8 @@ +cython +hypothesis +numpy +pytest +pytest-asyncio +pytz +python-dateutil +versioneer From 1a5a83597912c5fdce68f2899fa1020f74d11e79 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Wed, 28 Dec 2022 19:24:47 -0800 Subject: [PATCH 7/8] replace personal account with pandas org --- doc/source/development/contributing_environment.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/source/development/contributing_environment.rst b/doc/source/development/contributing_environment.rst index bfa698f27b13b..815a161557f48 100644 --- a/doc/source/development/contributing_environment.rst +++ b/doc/source/development/contributing_environment.rst @@ -171,7 +171,7 @@ If you are a new user and the image size is no concern to you, we suggest opting that includes all of the dependencies, as this will ensure you can run the test suite without any caveats. -To use any of the images, you should first start with ``docker pull willayd/pandas-dev:``, +To use any of the images, you should first start with ``docker pull pandas/pandas:``, where tag is one of *alpine*, *pip-minimal*, *mamba-minimal*, *pip-all* or *mamba-all*. You can then run the image without any extra configuration. @@ -180,15 +180,15 @@ you would run: .. code-block:: bash - docker pull willayd/pandas-dev:pip-all - docker run --rm -it -v ${PWD}:/home/pandas willayd/pandas-dev:pip-all + docker pull pandas/pandas:pip-all + docker run --rm -it -v ${PWD}:/home/pandas pandas/pandas:pip-all Similarly for *mamba-all* .. code-block:: bash - docker pull willayd/pandas-dev:mamba-all - docker run --rm -it -v ${PWD}:/home/pandas willayd/pandas-dev:mamba-all + docker pull pandas/pandas:mamba-all + docker run --rm -it -v ${PWD}:/home/pandas pandas/pandas:mamba-all The *mamba-* images will automatically activate the appropriate virtual environment for you on entry. From efb44b105f79b142a1095e9edf9306ba602e4699 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Wed, 28 Dec 2022 20:33:02 -0800 Subject: [PATCH 8/8] Removed docker CI job --- .github/workflows/code-checks.yml | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/.github/workflows/code-checks.yml b/.github/workflows/code-checks.yml index 280b6ed601f08..c4fc3575fdfa8 100644 --- a/.github/workflows/code-checks.yml +++ b/.github/workflows/code-checks.yml @@ -133,33 +133,6 @@ jobs: asv machine --yes asv run --quick --dry-run --strict --durations=30 --python=same - build_docker_dev_environment: - name: Build Docker Dev Environment - runs-on: ubuntu-22.04 - defaults: - run: - shell: bash -el {0} - - concurrency: - # https://github.community/t/concurrecy-not-work-for-push/183068/7 - group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-build_docker_dev_environment - cancel-in-progress: true - - steps: - - name: Clean up dangling images - run: docker image prune -f - - - name: Checkout - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Build image - run: docker build --pull --no-cache --tag pandas-dev-env . - - - name: Show environment - run: docker run --rm pandas-dev-env python -c "import pandas as pd; print(pd.show_versions())" - requirements-dev-text-installable: name: Test install requirements-dev.txt runs-on: ubuntu-22.04