From 240fe7878e0a1e45a4a6c4b75190fc6ef53cbc15 Mon Sep 17 00:00:00 2001 From: Anthony Johnson Date: Wed, 5 Sep 2018 16:52:46 -0600 Subject: [PATCH 1/5] Add a contrib Dockerfile for local build image on Linux This is necessary as permissions are all incorrect on the paths that are shared between the host system and the Docker build container. Closes #2692 --- contrib/Dockerfile | 17 +++++++++++++++++ contrib/docker_build.sh | 16 ++++++++++++++++ contrib/readme.rst | 14 +++++++++++++- docs/development/buildenvironments.rst | 20 ++++++++++++++++++++ 4 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 contrib/Dockerfile create mode 100755 contrib/docker_build.sh diff --git a/contrib/Dockerfile b/contrib/Dockerfile new file mode 100644 index 00000000000..233790ecace --- /dev/null +++ b/contrib/Dockerfile @@ -0,0 +1,17 @@ +FROM readthedocs/build:4.0rc1 + +ARG username +ARG uid=1000 +ARG gid=$uid +ARG label=latest + +ENV USERNAME ${username} +ENV UID ${uid} +ENV GID ${gid} + +USER root +RUN groupadd --gid $GID $USERNAME +RUN useradd -m --uid $UID --gid $GID $USERNAME +USER $username + +CMD ["/bin/bash"] diff --git a/contrib/docker_build.sh b/contrib/docker_build.sh new file mode 100755 index 00000000000..ef043c1de04 --- /dev/null +++ b/contrib/docker_build.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +username=`id -nu` +uid=`id -u` +gid=`id -g` + +version=$1 +[ -n "${version}" ] || version="latest" + +docker build \ + -t readthedocs/build:dev \ + --build-arg username=${username} \ + --build-arg uid=${uid} \ + --build-arg gid=${gid} \ + --build-arg label=${version} \ + . diff --git a/contrib/readme.rst b/contrib/readme.rst index dfa46ae0e6d..5b8198392cf 100644 --- a/contrib/readme.rst +++ b/contrib/readme.rst @@ -1,5 +1,17 @@ +Contrib +======= + +Building development Docker image +--------------------------------- + +If you run Linux, you likely need to build a local Docker image that extends our +default image:: + + cd contrib/ + ./docker_build.sh + Running Read the Docs via Supervisord -===================================== +------------------------------------- This is the easiest way to start all of the commands you'll need for development in an environment relatively close to the production evironment. All you need is diff --git a/docs/development/buildenvironments.rst b/docs/development/buildenvironments.rst index a6804680ffb..98a83f52007 100644 --- a/docs/development/buildenvironments.rst +++ b/docs/development/buildenvironments.rst @@ -82,3 +82,23 @@ DOCKER_VERSION Version of the API to use for the Docker API client. Default: :djangosetting:`DOCKER_VERSION` + +Local development +----------------- + +On Linux development environments, it's likely that your UID and GID do not +match the ``docs`` user that is set up as the default user for builds. In this +case, it's necessary to make a new image that overrides this user:: + + cd contrib/ + ./docker_build.sh + +You can also specify a specific image version to use:: + + cd contrib/ + ./docker_build.sh 4.0rc1 + +This will create a new image, ``readthedocs/build:dev``, which you can specify +as the default image in local settings overrides:: + + DOCKER_IMAGE = 'readthedocs/build:dev' From 22b53d0313c44838406d6ae14db3614cbc3f7c19 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Fri, 17 May 2019 14:46:00 +0200 Subject: [PATCH 2/5] Use always latest image to build the Docker image --- contrib/Dockerfile | 12 +++++------- contrib/docker_build.sh | 7 +------ docs/development/buildenvironments.rst | 12 ++++-------- 3 files changed, 10 insertions(+), 21 deletions(-) diff --git a/contrib/Dockerfile b/contrib/Dockerfile index 233790ecace..81b735907ca 100644 --- a/contrib/Dockerfile +++ b/contrib/Dockerfile @@ -1,17 +1,15 @@ -FROM readthedocs/build:4.0rc1 +FROM readthedocs/build:latest ARG username -ARG uid=1000 +ARG uid=$uid ARG gid=$uid -ARG label=latest -ENV USERNAME ${username} ENV UID ${uid} ENV GID ${gid} USER root -RUN groupadd --gid $GID $USERNAME -RUN useradd -m --uid $UID --gid $GID $USERNAME -USER $username +RUN groupadd --gid ${GID} docscustom +RUN usermod --uid ${UID} --gid ${GID} docs +USER docs CMD ["/bin/bash"] diff --git a/contrib/docker_build.sh b/contrib/docker_build.sh index ef043c1de04..7ca73ae78bc 100755 --- a/contrib/docker_build.sh +++ b/contrib/docker_build.sh @@ -4,13 +4,8 @@ username=`id -nu` uid=`id -u` gid=`id -g` -version=$1 -[ -n "${version}" ] || version="latest" - docker build \ - -t readthedocs/build:dev \ - --build-arg username=${username} \ + -t readthedocs/build:latest-dev \ --build-arg uid=${uid} \ --build-arg gid=${gid} \ - --build-arg label=${version} \ . diff --git a/docs/development/buildenvironments.rst b/docs/development/buildenvironments.rst index 98a83f52007..18fb9ef3e07 100644 --- a/docs/development/buildenvironments.rst +++ b/docs/development/buildenvironments.rst @@ -83,6 +83,7 @@ DOCKER_VERSION Default: :djangosetting:`DOCKER_VERSION` + Local development ----------------- @@ -93,12 +94,7 @@ case, it's necessary to make a new image that overrides this user:: cd contrib/ ./docker_build.sh -You can also specify a specific image version to use:: - - cd contrib/ - ./docker_build.sh 4.0rc1 - -This will create a new image, ``readthedocs/build:dev``, which you can specify -as the default image in local settings overrides:: +This will create a new image, ``readthedocs/build:latest-dev``, +which you can re-tag as ``latest`` to make Read the Docs use this image instead:: - DOCKER_IMAGE = 'readthedocs/build:dev' + docker tag readthedocs/build:latest-dev readthedocs/build:latest From 6ad9930921a2d209797b83fd6c43e86caf822b17 Mon Sep 17 00:00:00 2001 From: Anthony Johnson Date: Thu, 6 Jun 2019 13:24:45 -0600 Subject: [PATCH 3/5] Make image label configurable in docker build script This re-reverts back to using label for some of the dockerfile. Also, instead of creating a group, we simply groupmod the existing docs group. This doesn't address the configuration changes necessary for development images yet. --- contrib/Dockerfile | 11 ++++++----- contrib/docker_build.sh | 8 ++++++-- docs/development/buildenvironments.rst | 11 +++++++---- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/contrib/Dockerfile b/contrib/Dockerfile index 81b735907ca..216a4414bcc 100644 --- a/contrib/Dockerfile +++ b/contrib/Dockerfile @@ -1,14 +1,15 @@ -FROM readthedocs/build:latest +ARG label -ARG username -ARG uid=$uid -ARG gid=$uid +FROM readthedocs/build:${label} + +ARG uid +ARG gid ENV UID ${uid} ENV GID ${gid} USER root -RUN groupadd --gid ${GID} docscustom +RUN groupmod --gid ${GID} docs RUN usermod --uid ${UID} --gid ${GID} docs USER docs diff --git a/contrib/docker_build.sh b/contrib/docker_build.sh index 7ca73ae78bc..bc8b32142e6 100755 --- a/contrib/docker_build.sh +++ b/contrib/docker_build.sh @@ -1,11 +1,15 @@ #!/bin/sh -username=`id -nu` uid=`id -u` gid=`id -g` +version=$1 +[ -n "${version}" ] || version="latest" + docker build \ - -t readthedocs/build:latest-dev \ + --no-cache \ + -t readthedocs/build-dev:${version} \ --build-arg uid=${uid} \ --build-arg gid=${gid} \ + --build-arg label=${version} \ . diff --git a/docs/development/buildenvironments.rst b/docs/development/buildenvironments.rst index 18fb9ef3e07..39d8c785057 100644 --- a/docs/development/buildenvironments.rst +++ b/docs/development/buildenvironments.rst @@ -92,9 +92,12 @@ match the ``docs`` user that is set up as the default user for builds. In this case, it's necessary to make a new image that overrides this user:: cd contrib/ - ./docker_build.sh + ./docker_build.sh latest -This will create a new image, ``readthedocs/build:latest-dev``, -which you can re-tag as ``latest`` to make Read the Docs use this image instead:: +This will create a new image, ``readthedocs/build-dev:latest``. To build a +different image, you can instead specify a version to build:: - docker tag readthedocs/build:latest-dev readthedocs/build:latest + cd contrib/ + ./docker_build.sh 5.0 + +This will create a new image, ``readthedocs/build-dev:5.0``. From 16c5ac3dff6e1ab06a3674b899578d35fd9a4d77 Mon Sep 17 00:00:00 2001 From: Anthony Johnson Date: Tue, 11 Jun 2019 15:26:32 -0600 Subject: [PATCH 4/5] Update command call for docker_build.sh --- contrib/docker_build.sh | 4 +++- contrib/readme.rst | 3 +-- docs/development/buildenvironments.rst | 6 ++---- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/contrib/docker_build.sh b/contrib/docker_build.sh index bc8b32142e6..a0dde12d63f 100755 --- a/contrib/docker_build.sh +++ b/contrib/docker_build.sh @@ -2,6 +2,8 @@ uid=`id -u` gid=`id -g` +basedir=`dirname "$0"` +dockerfile=${basedir}/Dockerfile version=$1 [ -n "${version}" ] || version="latest" @@ -12,4 +14,4 @@ docker build \ --build-arg uid=${uid} \ --build-arg gid=${gid} \ --build-arg label=${version} \ - . + - <${dockerfile} diff --git a/contrib/readme.rst b/contrib/readme.rst index 5b8198392cf..ccc3a6a898c 100644 --- a/contrib/readme.rst +++ b/contrib/readme.rst @@ -7,8 +7,7 @@ Building development Docker image If you run Linux, you likely need to build a local Docker image that extends our default image:: - cd contrib/ - ./docker_build.sh + contrib/docker_build.sh latest Running Read the Docs via Supervisord ------------------------------------- diff --git a/docs/development/buildenvironments.rst b/docs/development/buildenvironments.rst index 39d8c785057..865a77e8698 100644 --- a/docs/development/buildenvironments.rst +++ b/docs/development/buildenvironments.rst @@ -91,13 +91,11 @@ On Linux development environments, it's likely that your UID and GID do not match the ``docs`` user that is set up as the default user for builds. In this case, it's necessary to make a new image that overrides this user:: - cd contrib/ - ./docker_build.sh latest + contrib/docker_build.sh latest This will create a new image, ``readthedocs/build-dev:latest``. To build a different image, you can instead specify a version to build:: - cd contrib/ - ./docker_build.sh 5.0 + contrib/docker_build.sh 5.0 This will create a new image, ``readthedocs/build-dev:5.0``. From 3d9a84c08ee2c8576a9bbc63e4488c4649f4e787 Mon Sep 17 00:00:00 2001 From: Anthony Johnson Date: Tue, 11 Jun 2019 15:52:33 -0600 Subject: [PATCH 5/5] Add configuration option that patches the docker image names in development This setting can be used from local_settings.py, and signifies that you have run ``docker_build.sh`` to build local development images that replace the UID/GID. --- docs/development/buildenvironments.rst | 11 ++++++++++- readthedocs/settings/dev.py | 11 +++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/docs/development/buildenvironments.rst b/docs/development/buildenvironments.rst index 865a77e8698..b47c309736f 100644 --- a/docs/development/buildenvironments.rst +++ b/docs/development/buildenvironments.rst @@ -89,7 +89,8 @@ Local development On Linux development environments, it's likely that your UID and GID do not match the ``docs`` user that is set up as the default user for builds. In this -case, it's necessary to make a new image that overrides this user:: +case, it's necessary to make a new image that overrides the UID and GID for the +normal container user:: contrib/docker_build.sh latest @@ -99,3 +100,11 @@ different image, you can instead specify a version to build:: contrib/docker_build.sh 5.0 This will create a new image, ``readthedocs/build-dev:5.0``. + +You can set a ``local_settings.py`` option to automatically patch the image +names to the development image names that are built here: + +DOCKER_USE_DEV_IMAGES + If set to ``True``, replace the normal Docker image name used in building + ``readthedocs/build`` with the image name output for these commands, + ``readthedocs/build-dev``. diff --git a/readthedocs/settings/dev.py b/readthedocs/settings/dev.py index 48c6e13ae36..48933f753a3 100644 --- a/readthedocs/settings/dev.py +++ b/readthedocs/settings/dev.py @@ -83,3 +83,14 @@ def MIDDLEWARE(self): from .local_settings import * # noqa except ImportError: pass + +# Allow for local settings override to trigger images name change +try: + if DOCKER_USE_DEV_IMAGES: + DOCKER_IMAGE_SETTINGS = { + key.replace('readthedocs/build:', 'readthedocs/build-dev:'): settings + for (key, settings) + in DOCKER_IMAGE_SETTINGS.items() + } +except NameError: + pass