From eb14034c4f53b468ad10257ebdc255479fa96dbe Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Sat, 30 May 2020 09:26:40 +0100 Subject: [PATCH 01/12] WIP: scripts for Windows host --- Dockerfile | 3 +- Makefile | 16 ++++++-- README.md | 2 + docker-files/windows/Dockerfile | 18 +++++++++ scripts/build-docs.sh | 8 ++-- windows.md | 69 +++++++++++++++++++++++++++++++++ 6 files changed, 107 insertions(+), 9 deletions(-) create mode 100644 docker-files/windows/Dockerfile create mode 100644 windows.md diff --git a/Dockerfile b/Dockerfile index 16f42aa..fe92648 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,5 +6,4 @@ RUN apt-get update && \ RUN conda update conda -y \ && conda install -y conda-build conda-verify gcc_linux-64 gxx_linux-64 \ && conda clean --all - -COPY pandas /pandas + \ No newline at end of file diff --git a/Makefile b/Makefile index d37edb8..9e60e26 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # TO EDIT -TAG ?= v1.0.3 +TAG ?= v1.0.4 -GH_USERNAME ?= TomAugspurger +GH_USERNAME ?= simonjayhawkins PANDAS_VERSION=$(TAG:v%=%) PANDAS_BASE_VERSION=$(shell echo $(PANDAS_VERSION) | awk -F '.' '{OFS="."} { print $$1, $$2}') TARGZ=pandas-$(PANDAS_VERSION).tar.gz @@ -59,6 +59,8 @@ pandas/dist/$(TARGZ): pandas-build \ sh /scripts/build_sdist.sh +# docker run -it --rm --name=pandas-sdist-build -v C:\Users\simon\pandas-release\scripts:/scripts -v C:\Users\simon\pandas-release\pandas:/pandas pandas-build /bin/bash + # ----------------------------------------------------------------------------- # Tests # These can be done in parallel @@ -73,6 +75,8 @@ conda-test: pandas-build \ sh -c "conda build --numpy=1.17.3 --python=3.8 /recipe --output-folder=/pandas/dist" +# docker run -it --rm --name=pandas-conda-test --env PANDAS_VERSION=1.0.4 -v C:\Users\simon\pandas-release\pandas:/pandas -v C:\Users\simon\pandas-release\recipe:/recipe pandas-build + pip-test: pandas/dist/$(TARGZ) docker run -it --rm \ --name=pandas-pip-test \ @@ -80,6 +84,8 @@ pip-test: pandas/dist/$(TARGZ) -v ${CURDIR}/scripts/pip_test.sh:/pip_test.sh \ pandas-build /bin/bash /pip_test.sh /pandas/dist/$(TARGZ) +# docker run -it --rm --name=pandas-pip-test -v C:\Users\simon\pandas-release\scripts:/scripts -v C:\Users\simon\pandas-release\pandas:/pandas pandas-build /bin/bash + # ----------------------------------------------------------------------------- # Docs # ----------------------------------------------------------------------------- @@ -92,6 +98,8 @@ doc: pandas-docs \ /build-docs.sh +# docker run -it --name=pandas-docs -v C:\Users\simon\pandas-release\pandas:/pandas -v C:\Users\simon\pandas-release\scripts:/scripts pandas-docs +# docker start pandas-docs -i upload-doc: rsync -rv -e ssh pandas/doc/build/html/ pandas.pydata.org:/usr/share/nginx/pandas/pandas-docs/version/$(PANDAS_VERSION)/ @@ -135,9 +143,11 @@ wheels: download-wheels: - ./scripts/download_wheels.py $(PANDAS_VERSION) + python ./scripts/download_wheels.py $(PANDAS_VERSION) # TODO: Fetch from https://www.lfd.uci.edu/~gohlke/pythonlibs/ upload-pypi: twine upload pandas/dist/pandas-$(PANDAS_VERSION)*.{whl,tar.gz} --skip-existing + +# twine upload pandas/dist/pandas-1.0.4*.{whl,tar.gz} --skip-existing \ No newline at end of file diff --git a/README.md b/README.md index 060a502..d972fa0 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ Release automation for pandas. +Windows users should follow the steps in `windows.md` + ## Steps to a release - [ ] Manually update diff --git a/docker-files/windows/Dockerfile b/docker-files/windows/Dockerfile new file mode 100644 index 0000000..4990f1c --- /dev/null +++ b/docker-files/windows/Dockerfile @@ -0,0 +1,18 @@ +FROM continuumio/miniconda3:latest + +RUN apt-get update && apt-get install make && apt-get clean + +RUN conda update conda -y + +RUN git clone https://github.com/pandas-dev/pandas-release + +RUN conda env update -n base --file=pandas-release/environment.yml + +ARG GH_USERNAME + +ENV GH_USERNAME=$GH_USERNAME + +WORKDIR pandas-release + +RUN make init-repos + diff --git a/scripts/build-docs.sh b/scripts/build-docs.sh index 7991753..3b271ee 100755 --- a/scripts/build-docs.sh +++ b/scripts/build-docs.sh @@ -15,7 +15,7 @@ python setup.py build_ext -i -j 4 cd /pandas/doc -./make.py html -./make.py zip_html -./make.py latex_forced ||: -./make.py latex_forced +python ./make.py html +python ./make.py zip_html +python ./make.py latex_forced ||: +python ./make.py latex_forced diff --git a/windows.md b/windows.md new file mode 100644 index 0000000..44053dd --- /dev/null +++ b/windows.md @@ -0,0 +1,69 @@ + + +install docker +ensure the following repositories are forked to your GitHub account + - https://github.com/pandas-dev/pandas-website + - https://github.com/conda-forge/pandas-feedstock + - https://github.com/MacPython/pandas-wheels + - https://github.com/pandas-dev/pandas + + +build a docker image with the conda base environment configured and repositories initialized. + +todo: use url for dockerfile to save cloning pandas-release to host machine + +change GH_USERNAME to your Github username + +``` +docker build -t pandas-release --build-arg GH_USERNAME=simonjayhawkins -f docker-files/windows/Dockerfile . +``` + +next we will prepare a container for interactive use during the release process and copy the repositories +into a shared volume for building the distribution and testing. + +change TAG to the release version + +``` +docker run -it --env TAG=v1.0.4 --name=pandas-release -v pandas-release:/pandas-release pandas-release /bin/bash +``` + +the docker container should be now be running + +make sure the repos are up-to-date + +``` +make update-repos +``` + +configure email and name for git # todo: get from host + +``` +git config --global user.email "you@example.com" +git config --global user.name "Your Name" +``` + +Tag the release +``` +make tag +``` + +stop the container +""" +exit +""" + +create the build container + +docker build -t pandas-build https://raw.githubusercontent.com/pandas-dev/pandas-release/master/Dockerfile + +build the sdist + + +docker run --rm -v pandas-release:/pandas-release pandas-build /bin/bash /pandas-release/scripts/build_sdist.sh +docker run --rm -v pandas-release:/pandas-release pandas-build -w /pandas-release /bin/bash /scripts/build_sdist.sh + +restart the release container + +``` +docker start pandas-release -i +``` From ca5392999418034c1f28c88f6e93cfa768f1de76 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Tue, 16 Jun 2020 13:10:37 +0100 Subject: [PATCH 02/12] wip - was using volume, will change to mount cloned directory --- Makefile | 2 +- windows.md | 40 ++++++++++++++++++++++++++++++++-------- 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 9e60e26..aba59f9 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # TO EDIT -TAG ?= v1.0.4 +TAG ?= v1.0.5 GH_USERNAME ?= simonjayhawkins PANDAS_VERSION=$(TAG:v%=%) diff --git a/windows.md b/windows.md index 44053dd..08cba76 100644 --- a/windows.md +++ b/windows.md @@ -10,7 +10,7 @@ ensure the following repositories are forked to your GitHub account build a docker image with the conda base environment configured and repositories initialized. -todo: use url for dockerfile to save cloning pandas-release to host machine +TODO: use url for dockerfile to save cloning pandas-release to host machine change GH_USERNAME to your Github username @@ -24,43 +24,67 @@ into a shared volume for building the distribution and testing. change TAG to the release version ``` -docker run -it --env TAG=v1.0.4 --name=pandas-release -v pandas-release:/pandas-release pandas-release /bin/bash +docker run -it --env TAG=v1.0.5 --name=pandas-release -v pandas-release:/pandas-release pandas-release /bin/bash ``` the docker container should be now be running make sure the repos are up-to-date +TODO: also make sure conda environment is up-to-date and pandas-release repo is up-to-date if +re-using an older Docker image (maybe need another script in Makefile) +NOTE: safer to build new image if not on metered or slow internet connection. ``` make update-repos ``` -configure email and name for git # todo: get from host +configure email and name for git +TODO: get from host or better still use Pandas Development Team as default in the pandas-release docker image build ``` git config --global user.email "you@example.com" git config --global user.name "Your Name" ``` -Tag the release +Tag the release (This doesn't push the tag) ``` make tag ``` stop the container -""" +``` exit -""" +``` create the build container +NOTE: this is the make docker-image step if not using Windows +NOTE: Step 4/4 : COPY pandas /pandas fails until pandas-release/master updated + +``` docker build -t pandas-build https://raw.githubusercontent.com/pandas-dev/pandas-release/master/Dockerfile +``` build the sdist +NOTE: this is the make pandas/dist/<>.tar.gz step if not using Windows +TODO: create symlinks so this can be automated + +``` +# docker run --rm -v pandas-release:/pandas-release pandas-build /bin/bash /pandas-release/scripts/build_sdist.sh + +docker run -it --rm -v pandas-release:/pandas-release pandas-build /bin/bash + +rm -r pandas + +ln -s pandas-release/pandas pandas + +cd pandas-release/ -docker run --rm -v pandas-release:/pandas-release pandas-build /bin/bash /pandas-release/scripts/build_sdist.sh -docker run --rm -v pandas-release:/pandas-release pandas-build -w /pandas-release /bin/bash /scripts/build_sdist.sh +./scripts/build_sdist.sh + +exit +``` restart the release container From 129a8ed196fb11d9d97dd9d377b0fcc0ce1bfe15 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Tue, 16 Jun 2020 19:38:32 +0100 Subject: [PATCH 03/12] wip --- Makefile | 12 +--- docker-files/windows/Dockerfile | 3 + scripts/build-docs.sh | 8 +-- scripts/pip_test.sh | 2 +- windows.md | 119 +++++++++++++++++++++++++++----- 5 files changed, 110 insertions(+), 34 deletions(-) diff --git a/Makefile b/Makefile index aba59f9..d132b18 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # TO EDIT -TAG ?= v1.0.5 +TAG ?= v1.0.3 -GH_USERNAME ?= simonjayhawkins +GH_USERNAME ?= TomAugspurger PANDAS_VERSION=$(TAG:v%=%) PANDAS_BASE_VERSION=$(shell echo $(PANDAS_VERSION) | awk -F '.' '{OFS="."} { print $$1, $$2}') TARGZ=pandas-$(PANDAS_VERSION).tar.gz @@ -59,8 +59,6 @@ pandas/dist/$(TARGZ): pandas-build \ sh /scripts/build_sdist.sh -# docker run -it --rm --name=pandas-sdist-build -v C:\Users\simon\pandas-release\scripts:/scripts -v C:\Users\simon\pandas-release\pandas:/pandas pandas-build /bin/bash - # ----------------------------------------------------------------------------- # Tests # These can be done in parallel @@ -75,8 +73,6 @@ conda-test: pandas-build \ sh -c "conda build --numpy=1.17.3 --python=3.8 /recipe --output-folder=/pandas/dist" -# docker run -it --rm --name=pandas-conda-test --env PANDAS_VERSION=1.0.4 -v C:\Users\simon\pandas-release\pandas:/pandas -v C:\Users\simon\pandas-release\recipe:/recipe pandas-build - pip-test: pandas/dist/$(TARGZ) docker run -it --rm \ --name=pandas-pip-test \ @@ -84,8 +80,6 @@ pip-test: pandas/dist/$(TARGZ) -v ${CURDIR}/scripts/pip_test.sh:/pip_test.sh \ pandas-build /bin/bash /pip_test.sh /pandas/dist/$(TARGZ) -# docker run -it --rm --name=pandas-pip-test -v C:\Users\simon\pandas-release\scripts:/scripts -v C:\Users\simon\pandas-release\pandas:/pandas pandas-build /bin/bash - # ----------------------------------------------------------------------------- # Docs # ----------------------------------------------------------------------------- @@ -98,8 +92,6 @@ doc: pandas-docs \ /build-docs.sh -# docker run -it --name=pandas-docs -v C:\Users\simon\pandas-release\pandas:/pandas -v C:\Users\simon\pandas-release\scripts:/scripts pandas-docs -# docker start pandas-docs -i upload-doc: rsync -rv -e ssh pandas/doc/build/html/ pandas.pydata.org:/usr/share/nginx/pandas/pandas-docs/version/$(PANDAS_VERSION)/ diff --git a/docker-files/windows/Dockerfile b/docker-files/windows/Dockerfile index 4990f1c..04527dc 100644 --- a/docker-files/windows/Dockerfile +++ b/docker-files/windows/Dockerfile @@ -16,3 +16,6 @@ WORKDIR pandas-release RUN make init-repos +RUN git config --global user.email "pandas-dev@python.org" + +RUN git config --global user.name "Pandas Development Team" diff --git a/scripts/build-docs.sh b/scripts/build-docs.sh index 3b271ee..7991753 100755 --- a/scripts/build-docs.sh +++ b/scripts/build-docs.sh @@ -15,7 +15,7 @@ python setup.py build_ext -i -j 4 cd /pandas/doc -python ./make.py html -python ./make.py zip_html -python ./make.py latex_forced ||: -python ./make.py latex_forced +./make.py html +./make.py zip_html +./make.py latex_forced ||: +./make.py latex_forced diff --git a/scripts/pip_test.sh b/scripts/pip_test.sh index 8eb98ce..d0555bc 100755 --- a/scripts/pip_test.sh +++ b/scripts/pip_test.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -e -# Cytyhon is *not* required here. +# Cython is *not* required here. conda create -n pip-test -y python=3.7 numpy pytz python-dateutil pytest pytest-mock hypothesis nomkl source activate pip-test diff --git a/windows.md b/windows.md index 08cba76..e1f595e 100644 --- a/windows.md +++ b/windows.md @@ -10,8 +10,6 @@ ensure the following repositories are forked to your GitHub account build a docker image with the conda base environment configured and repositories initialized. -TODO: use url for dockerfile to save cloning pandas-release to host machine - change GH_USERNAME to your Github username ``` @@ -38,15 +36,8 @@ NOTE: safer to build new image if not on metered or slow internet connection. make update-repos ``` -configure email and name for git -TODO: get from host or better still use Pandas Development Team as default in the pandas-release docker image build - -``` -git config --global user.email "you@example.com" -git config --global user.name "Your Name" -``` - Tag the release (This doesn't push the tag) + ``` make tag ``` @@ -58,36 +49,126 @@ exit create the build container -NOTE: this is the make docker-image step if not using Windows -NOTE: Step 4/4 : COPY pandas /pandas fails until pandas-release/master updated - ``` -docker build -t pandas-build https://raw.githubusercontent.com/pandas-dev/pandas-release/master/Dockerfile +docker build -t pandas-build . ``` build the sdist -NOTE: this is the make pandas/dist/<>.tar.gz step if not using Windows -TODO: create symlinks so this can be automated +``` +docker run -it --rm -v pandas-release:/pandas-release pandas-build /bin/bash +ln -s pandas-release/pandas pandas + +cd pandas-release/ + +./scripts/build_sdist.sh + +exit ``` -# docker run --rm -v pandas-release:/pandas-release pandas-build /bin/bash /pandas-release/scripts/build_sdist.sh +pip tests + +``` docker run -it --rm -v pandas-release:/pandas-release pandas-build /bin/bash -rm -r pandas +ln -s pandas-release/pandas pandas + +cd pandas-release/ + +./scripts/pip_test.sh /pandas/dist/pandas-1.0.5.tar.gz + +exit + +``` + +conda tests + +``` +docker run -it --rm --env PANDAS_VERSION=1.0.5 -v pandas-release:/pandas-release pandas-build /bin/bash ln -s pandas-release/pandas pandas cd pandas-release/ -./scripts/build_sdist.sh +conda build --numpy=1.17.3 --python=3.8 ./recipe --output-folder=/pandas/dist exit + ``` +copy the sdist file to the local host + +``` +docker run -it --rm -v %cd%:/local -v pandas-release:/pandas-release pandas-release /bin/bash -c "cp /pandas-release/pandas/dist/pandas-1.0.5.tar.gz /local/" +``` + +Push the tag. No going back now. + restart the release container ``` docker start pandas-release -i + +... + +On pandas you should also now create and tag a new branch, so + +... +``` + +Now manually create a release https://github.com/pandas-dev/pandas/releases + +Make sure to upload the sdist as the "binary". Conda-forge uses it. + + +Start the binary builds. + +restart the release container + +``` +docker start pandas-release -i + +make conda-forge + +make wheels + +exit +``` + +Open PRs for each of those. + +Note that `make wheels` actually pushes a job to MacPython to produce wheels which we will download later. + + +Docs. You can cheat and re-tag / rebuild these if needed. + + + +TODO build an intermediate doc image (and why pandas conda env not in docker image?) +``` +docker run -it --name=pandas-docs -v pandas-release:/pandas-release pandas-docs /bin/bash + +rm -r pandas + +ln -s pandas-release/pandas pandas + +cd pandas-release/ + +# following maybe necessary to prevent segfaults +conda update -n base -c defaults conda + +conda env create --file=/pandas/environment.yml --name=pandas + +... + +./scripts/build-docs.sh + +... ``` From cce6e8ae8dbe5c59ed8384a5af6ea8eae086e819 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Wed, 17 Jun 2020 10:08:04 +0100 Subject: [PATCH 04/12] wip --- windows.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/windows.md b/windows.md index e1f595e..626eb9d 100644 --- a/windows.md +++ b/windows.md @@ -166,9 +166,14 @@ conda update -n base -c defaults conda conda env create --file=/pandas/environment.yml --name=pandas -... - ./scripts/build-docs.sh -... +exit ``` + +copy the built doc files to host and manually inspect html and pdf docs. + +``` +docker run -it --rm -v %cd%:/local -v pandas-release:/pandas-release pandas-release /bin/bash -c "cp -r /pandas-release/pandas/doc/build/ /local/pandas-docs" +``` + From f6ab739bcef8b323ae9b1feaf9d3b38abbc82038 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Wed, 17 Jun 2020 16:27:33 +0100 Subject: [PATCH 05/12] wip --- windows.md | 43 ++++++++++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 11 deletions(-) diff --git a/windows.md b/windows.md index 626eb9d..6a5ebdd 100644 --- a/windows.md +++ b/windows.md @@ -1,6 +1,7 @@ -install docker +install Docker + ensure the following repositories are forked to your GitHub account - https://github.com/pandas-dev/pandas-website - https://github.com/conda-forge/pandas-feedstock @@ -8,7 +9,11 @@ ensure the following repositories are forked to your GitHub account - https://github.com/pandas-dev/pandas -build a docker image with the conda base environment configured and repositories initialized. +Open an Anaconda Prompt +TODO: resolve git bash Docker volume issues so that make can be used on host + + +build a Docker image with the conda base environment configured and repositories initialized. change GH_USERNAME to your Github username @@ -16,8 +21,14 @@ change GH_USERNAME to your Github username docker build -t pandas-release --build-arg GH_USERNAME=simonjayhawkins -f docker-files/windows/Dockerfile . ``` -next we will prepare a container for interactive use during the release process and copy the repositories -into a shared volume for building the distribution and testing. +next we will prepare a Docker container for interactive use during the release process and copy the repositories +from the Docker image into a Docker volume for building the distribution and testing. + +to start with a clean volume (after a previous release) + +``` +docker volume rm pandas-release +``` change TAG to the release version @@ -25,12 +36,11 @@ change TAG to the release version docker run -it --env TAG=v1.0.5 --name=pandas-release -v pandas-release:/pandas-release pandas-release /bin/bash ``` -the docker container should be now be running +the Docker release container should be now be running make sure the repos are up-to-date TODO: also make sure conda environment is up-to-date and pandas-release repo is up-to-date if -re-using an older Docker image (maybe need another script in Makefile) -NOTE: safer to build new image if not on metered or slow internet connection. +re-using an older Docker image ``` make update-repos @@ -47,7 +57,8 @@ stop the container exit ``` -create the build container +create the Docker image for the sdist build, pip test and conda test containers +TODO: maybe update the image with apt-get for cached build ``` docker build -t pandas-build . @@ -55,6 +66,9 @@ docker build -t pandas-build . build the sdist +TODO: some of the next steps are repetative. set WORKDIR and symlink to /pandas in pandas-build Docker image instead +TODO: add container name (as in Makefile) and do not destroy container on exit + ``` docker run -it --rm -v pandas-release:/pandas-release pandas-build /bin/bash @@ -69,6 +83,9 @@ exit pip tests +TODO: avoid need to pass explicit filename below +TODO: add container name (as in Makefile) and do not destroy container on exit + ``` docker run -it --rm -v pandas-release:/pandas-release pandas-build /bin/bash @@ -84,6 +101,9 @@ exit conda tests +TODO: add container name (as in Makefile) and do not destroy container on exit +TODO: avoid need to re-type version below + ``` docker run -it --rm --env PANDAS_VERSION=1.0.5 -v pandas-release:/pandas-release pandas-build /bin/bash @@ -97,10 +117,11 @@ exit ``` -copy the sdist file to the local host +copy the sdist file from the Docker volume to the local host +TODO: avoid need to enter specific filename below (maybe just copy contents of dist directory instead) ``` -docker run -it --rm -v %cd%:/local -v pandas-release:/pandas-release pandas-release /bin/bash -c "cp /pandas-release/pandas/dist/pandas-1.0.5.tar.gz /local/" +docker run -t --rm -v %cd%:/local -v pandas-release:/pandas-release pandas-release /bin/bash -c "cp /pandas-release/pandas/dist/pandas-1.0.5.tar.gz /local/" ``` Push the tag. No going back now. @@ -151,7 +172,7 @@ Docs. You can cheat and re-tag / rebuild these if needed. pandas-docs \ /build-docs.sh --> -TODO build an intermediate doc image (and why pandas conda env not in docker image?) +TODO build an intermediate doc image (and why pandas conda env not in Docker image?) ``` docker run -it --name=pandas-docs -v pandas-release:/pandas-release pandas-docs /bin/bash From 1ef9852fd5a971ab1877a29f979b64910a7c4467 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Wed, 17 Jun 2020 16:34:02 +0100 Subject: [PATCH 06/12] whitespace --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index fe92648..d06874e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,4 +6,3 @@ RUN apt-get update && \ RUN conda update conda -y \ && conda install -y conda-build conda-verify gcc_linux-64 gxx_linux-64 \ && conda clean --all - \ No newline at end of file From 3608baedf9c27773b4693a5bf709210177d68938 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Wed, 17 Jun 2020 18:19:48 +0100 Subject: [PATCH 07/12] wip --- windows.md | 103 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 61 insertions(+), 42 deletions(-) diff --git a/windows.md b/windows.md index 6a5ebdd..3bebd6e 100644 --- a/windows.md +++ b/windows.md @@ -1,74 +1,82 @@ +# README for MS Windows Users +## Preparing for Your First Release -install Docker +Install Docker -ensure the following repositories are forked to your GitHub account +Ensure the following repositories are forked to your GitHub account - https://github.com/pandas-dev/pandas-website - https://github.com/conda-forge/pandas-feedstock - https://github.com/MacPython/pandas-wheels - https://github.com/pandas-dev/pandas - Open an Anaconda Prompt + +Build a Docker image with the conda base environment configured and repositories initialized. -build a Docker image with the conda base environment configured and repositories initialized. - -change GH_USERNAME to your Github username +**change GH_USERNAME to your Github username** ``` docker build -t pandas-release --build-arg GH_USERNAME=simonjayhawkins -f docker-files/windows/Dockerfile . ``` -next we will prepare a Docker container for interactive use during the release process and copy the repositories +## Preparing the Release Environment for a New Release + +Next we will prepare a Docker container for interactive use during the release process and copy the repositories from the Docker image into a Docker volume for building the distribution and testing. -to start with a clean volume (after a previous release) +To start with a clean volume. (after a previous release) ``` docker volume rm pandas-release ``` -change TAG to the release version - +**change TAG to the release version** ``` docker run -it --env TAG=v1.0.5 --name=pandas-release -v pandas-release:/pandas-release pandas-release /bin/bash ``` -the Docker release container should be now be running +The Docker release container should be now be running. -make sure the repos are up-to-date +Make sure the repos are up-to-date + ``` make update-repos ``` -Tag the release (This doesn't push the tag) +Tag the release. (This doesn't push the tag) ``` make tag ``` -stop the container +Stop the container. ``` exit ``` -create the Docker image for the sdist build, pip test and conda test containers -TODO: maybe update the image with apt-get for cached build +## Preparing the Build and Test Environment + +Create the Docker image for the sdist build, pip test and conda test containers + ``` docker build -t pandas-build . ``` -build the sdist - +## Build the sdist + ``` docker run -it --rm -v pandas-release:/pandas-release pandas-build /bin/bash @@ -81,10 +89,13 @@ cd pandas-release/ exit ``` -pip tests - +## Pip Tests + + +**change filename to the release version** ``` docker run -it --rm -v pandas-release:/pandas-release pandas-build /bin/bash @@ -99,10 +110,12 @@ exit ``` -conda tests - +## Conda Tests + + **change PANDAS_VERSION to the release version** ``` docker run -it --rm --env PANDAS_VERSION=1.0.5 -v pandas-release:/pandas-release pandas-build /bin/bash @@ -117,35 +130,47 @@ exit ``` -copy the sdist file from the Docker volume to the local host +## Copy the sdist File from the Docker Volume to the Local Host. + +**change filename to the release version** ``` docker run -t --rm -v %cd%:/local -v pandas-release:/pandas-release pandas-release /bin/bash -c "cp /pandas-release/pandas/dist/pandas-1.0.5.tar.gz /local/" ``` -Push the tag. No going back now. +## Push the Tag. -restart the release container +**No going back now.** +Restart the release container. + ``` docker start pandas-release -i -... +make push-tag +``` + +## Create New Branch +(not needed for Patch release) On pandas you should also now create and tag a new branch, so ... -``` + +## Create a Release Now manually create a release https://github.com/pandas-dev/pandas/releases Make sure to upload the sdist as the "binary". Conda-forge uses it. -Start the binary builds. +## Start the Binary Builds. -restart the release container +Restart the release container. ``` docker start pandas-release -i @@ -162,17 +187,11 @@ Open PRs for each of those. Note that `make wheels` actually pushes a job to MacPython to produce wheels which we will download later. -Docs. You can cheat and re-tag / rebuild these if needed. - - - +## Build the Docs. +You can cheat and re-tag / rebuild these if needed. + ``` docker run -it --name=pandas-docs -v pandas-release:/pandas-release pandas-docs /bin/bash @@ -192,7 +211,7 @@ conda env create --file=/pandas/environment.yml --name=pandas exit ``` -copy the built doc files to host and manually inspect html and pdf docs. +Copy the built doc files to host and manually inspect html and pdf docs. ``` docker run -it --rm -v %cd%:/local -v pandas-release:/pandas-release pandas-release /bin/bash -c "cp -r /pandas-release/pandas/doc/build/ /local/pandas-docs" From 126cbc1124dd8091e4814797ac31130a22c92a95 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Wed, 17 Jun 2020 18:39:04 +0100 Subject: [PATCH 08/12] revert changes to Makefile --- Makefile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Makefile b/Makefile index d132b18..d37edb8 100644 --- a/Makefile +++ b/Makefile @@ -135,11 +135,9 @@ wheels: download-wheels: - python ./scripts/download_wheels.py $(PANDAS_VERSION) + ./scripts/download_wheels.py $(PANDAS_VERSION) # TODO: Fetch from https://www.lfd.uci.edu/~gohlke/pythonlibs/ upload-pypi: twine upload pandas/dist/pandas-$(PANDAS_VERSION)*.{whl,tar.gz} --skip-existing - -# twine upload pandas/dist/pandas-1.0.4*.{whl,tar.gz} --skip-existing \ No newline at end of file From db7b75d2fa39831a7f1925682a790bb3aee7f4c5 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Wed, 17 Jun 2020 20:38:35 +0100 Subject: [PATCH 09/12] wip --- docker-files/windows/Dockerfile | 4 +++- windows.md | 24 +++++++++++++++++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/docker-files/windows/Dockerfile b/docker-files/windows/Dockerfile index 04527dc..836f159 100644 --- a/docker-files/windows/Dockerfile +++ b/docker-files/windows/Dockerfile @@ -1,6 +1,8 @@ FROM continuumio/miniconda3:latest -RUN apt-get update && apt-get install make && apt-get clean +RUN apt-get update && \ + apt-get install -y make rsync && \ + apt-get clean RUN conda update conda -y diff --git a/windows.md b/windows.md index 3bebd6e..4d3037d 100644 --- a/windows.md +++ b/windows.md @@ -213,7 +213,29 @@ exit Copy the built doc files to host and manually inspect html and pdf docs. +**first remove the local pandas-docs directory (just use file manager)** + +``` +docker run -t --rm -v %cd%:/local -v pandas-release:/pandas-release pandas-release /bin/bash -c "cp -r /pandas-release/pandas/doc/build/ /local/pandas-docs" +``` + +## Upload the docs + +restart the release container + ``` -docker run -it --rm -v %cd%:/local -v pandas-release:/pandas-release pandas-release /bin/bash -c "cp -r /pandas-release/pandas/doc/build/ /local/pandas-docs" +docker start pandas-release -i + +# TODO upload ssh keys + +make upload-doc + +# TODO add steps to update website + ``` +## Upload the Binarys to PyPI + + From 9cf96719bd5c6dad0952f9697212b09b63572c92 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Thu, 18 Jun 2020 11:43:52 +0100 Subject: [PATCH 10/12] wip --- Makefile | 1 - README.md | 10 +++------- windows.md | 37 ++++++++++++++++++++++++++++++------- 3 files changed, 33 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index d37edb8..d31f387 100644 --- a/Makefile +++ b/Makefile @@ -136,7 +136,6 @@ wheels: download-wheels: ./scripts/download_wheels.py $(PANDAS_VERSION) - # TODO: Fetch from https://www.lfd.uci.edu/~gohlke/pythonlibs/ upload-pypi: diff --git a/README.md b/README.md index d972fa0..5a3e9c8 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ Release automation for pandas. -Windows users should follow the steps in `windows.md` +**Windows users should follow the steps in `windows.md`** ## Steps to a release @@ -75,17 +75,14 @@ Docs. You can cheat and re-tag / rebuild these if needed. make doc ``` -Once the binaries finish, you'll need to manually upload the -wheels to PyPI. Assuming the job which `make wheels` triggered on MacPython completed successfully (you may want to double check this) -you can download a copy of the wheels for Mac / Linux locally. +Once the binaries finish, you'll need to manually upload the wheels to PyPI. +Assuming the job which `make wheels` triggered on MacPython completed successfully (you may want to double check this https://anaconda.org/multibuild-wheels-staging/pandas/files) you can download a copy of the wheels locally. ``` make download-wheels ``` -Christoph Gohlke builds the windows wheels. Fetch from from https://www.lfd.uci.edu/~gohlke/pythonlibs/ and download to the same `dist` directory. - Upload the wheels and sdist ``` @@ -130,7 +127,6 @@ goto announce. # Finalize -- [ ] Download Christoph's wheels and upload to PyPI - [ ] Announce Mailing List - [ ] Announce Twitter diff --git a/windows.md b/windows.md index 4d3037d..bd50117 100644 --- a/windows.md +++ b/windows.md @@ -152,6 +152,8 @@ TODO: does this need to be in interactive mode docker start pandas-release -i make push-tag + +exit ``` ## Create New Branch @@ -213,29 +215,50 @@ exit Copy the built doc files to host and manually inspect html and pdf docs. -**first remove the local pandas-docs directory (just use file manager)** +**first remove the local pandas-docs directory (just manually use file manager for now)** ``` docker run -t --rm -v %cd%:/local -v pandas-release:/pandas-release pandas-release /bin/bash -c "cp -r /pandas-release/pandas/doc/build/ /local/pandas-docs" ``` ## Upload the docs - -restart the release container + +Copy ssh key and config into release container and restart container ``` +docker cp %userprofile%/.ssh pandas-release:/root/.ssh + docker start pandas-release -i -# TODO upload ssh keys +chmod 400 ~/.ssh/id_rsa make upload-doc -# TODO add steps to update website - +exit ``` ## Upload the Binarys to PyPI +Once the binaries finish, you'll need to manually upload the wheels to PyPI. +Assuming the job which `make wheels` triggered on MacPython completed successfully (you may want to double check this https://anaconda.org/multibuild-wheels-staging/pandas/files) you can download a copy of the wheels locally. + +``` +docker start pandas-release -i + +make download-wheels + +make upload-pypi + +exit +``` + +## Finalize the docs + +... From 2dba9d6c145c2f45a0198134ff888ea75c3fff95 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Thu, 18 Jun 2020 13:32:37 +0100 Subject: [PATCH 11/12] delete web stuff --- Makefile | 13 ------------- README.md | 21 +-------------------- windows.md | 23 +++++++++++++++++++---- 3 files changed, 20 insertions(+), 37 deletions(-) diff --git a/Makefile b/Makefile index d31f387..5430a02 100644 --- a/Makefile +++ b/Makefile @@ -15,14 +15,12 @@ SHELL := /bin/bash init-repos: git clone https://github.com/pandas-dev/pandas && git -C pandas remote rename origin upstream && git -C pandas remote add origin https://github.com/$(GH_USERNAME)/pandas - git clone https://github.com/pandas-dev/pandas-website && git -C pandas-website remote rename origin upstream && git -C pandas-website remote add origin https://github.com/$(GH_USERNAME)/pandas-website git clone https://github.com/conda-forge/pandas-feedstock && git -C pandas-feedstock remote rename origin upstream && git -C pandas-feedstock remote add origin https://github.com/$(GH_USERNAME)/pandas-feedstock git clone --recursive https://github.com/MacPython/pandas-wheels && git -C pandas-wheels remote rename origin upstream && git -C pandas-wheels remote add origin https://github.com/$(GH_USERNAME)/pandas-wheels update-repos: git -C pandas checkout master && git -C pandas pull git -C pandas-wheels checkout master && git -C pandas-wheels pull - git -C pandas-website checkout master && git -C pandas-website pull git -C pandas-feedstock checkout master && git -C pandas-feedstock pull pushd pandas-wheels && git submodule update --recursive --remote && popd @@ -105,17 +103,6 @@ link-version: push-doc: | upload-doc link-stable link-version -website: - pushd pandas/web && \ - git checkout master && \ - git pull && \ - ./pandas_web.py pandas && \ - popd - - -push-website: - rsync -ravzI -e ssh pandas/web/build/* pandas.pydata.org:/usr/share/nginx/pandas/ - push-tag: pushd pandas && ../scripts/push-tag.py $(TAG) && popd diff --git a/README.md b/README.md index 5a3e9c8..0783348 100644 --- a/README.md +++ b/README.md @@ -93,34 +93,15 @@ Finalize the docs ``` make upload-doc -make website ``` -You also need to make edits to the pandas-website to appropriately display items. -Ideally this could be done via `make push-website` as the rule exists but the -intermediary steps aren't fully automated yet. +To make sure /stable and the latest minor revision point to the new release run the following. -```sh -pushd pandas-website -mv latest.rst previous.rst -# Recreate latest.rst to match release notes from GH in earlier steps -# Update pre_release.json and releases.json -git commit -am "Your updates" -git push -make html -make upload ``` - -To make sure /stable and the latest minor revision point to the new release run the following from root - -```sh -popd # should bring us back to root from pandas-website make link-stable make link-version ``` -Now check pandas.pydata.org and ensure the sidebar and links are correct! - goto announce. diff --git a/windows.md b/windows.md index bd50117..3962359 100644 --- a/windows.md +++ b/windows.md @@ -224,12 +224,12 @@ TODO: add steps to clean the pandas-docs directory from the docker container bef docker run -t --rm -v %cd%:/local -v pandas-release:/pandas-release pandas-release /bin/bash -c "cp -r /pandas-release/pandas/doc/build/ /local/pandas-docs" ``` -## Upload the docs +## Upload the Docs -Copy ssh key and config into release container and restart container +Copy ssh key and config into release container and restart container. ``` docker cp %userprofile%/.ssh pandas-release:/root/.ssh @@ -259,6 +259,21 @@ make upload-pypi exit ``` -## Finalize the docs +## Finalize the Docs -... +Do this once the wheels are available on PyPI. + +``` +docker start pandas-release -i + +make link-stable + +make link-version + +exit +``` + +# Announce + +- [ ] Announce Mailing List +- [ ] Announce Twitter From e181902ab0245d599cfe40145cf59214aa4b785c Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Thu, 18 Jun 2020 13:36:53 +0100 Subject: [PATCH 12/12] remove webiste repo link --- windows.md | 1 - 1 file changed, 1 deletion(-) diff --git a/windows.md b/windows.md index 3962359..37da179 100644 --- a/windows.md +++ b/windows.md @@ -5,7 +5,6 @@ Install Docker Ensure the following repositories are forked to your GitHub account - - https://github.com/pandas-dev/pandas-website - https://github.com/conda-forge/pandas-feedstock - https://github.com/MacPython/pandas-wheels - https://github.com/pandas-dev/pandas