Merge pull request #6418 from readthedocs/humitos/docker-tasks

inv tasks to use when developing with docker

Author: ericholscher

.gitignore

@@ -67,3 +67,6 @@ env.bak/
 venv.bak/
 
 dev.db-journal
+/readthedocs/settings/build.py
+/readthedocs/settings/celery.py
+/readthedocs/settings/web.py

docker-compose-search.yml

@@ -24,6 +24,9 @@ services:
       - storage
       - azure-cli
       - search
+    environment:
+      - DJANGO_SETTINGS_MODULE=readthedocs.settings.celery
+      - SEARCH=t
 
   # You may need to increase the memory for vm
   # sudo sysctl -w vm.max_map_count=262144

docker-compose.yml

@@ -17,7 +17,7 @@ services:
     # Image used for all the other services (proxito, web, celery, build)
     build:
       context: .
-      dockerfile: ${PWD}/docker/Dockerfile
+      dockerfile: ${PWD}/dockerfiles/Dockerfile
 
   nginx:
     image: nginx
@@ -28,16 +28,16 @@ services:
       - proxito
       - storage
     volumes:
-      - ${PWD}/docker/nginx:/etc/nginx/conf.d
+      - ${PWD}/dockerfiles/nginx:/etc/nginx/conf.d
 
   proxito:
     image: readthedocsorg_server:latest
     volumes:
-      - ${PWD}/docker/entrypoints/common.sh:/usr/src/app/docker/common.sh
-      - ${PWD}/docker/entrypoints/proxito.sh:/usr/src/app/docker/proxito.sh
+      - ${PWD}/dockerfiles/entrypoints/common.sh:/usr/src/app/docker/common.sh
+      - ${PWD}/dockerfiles/entrypoints/proxito.sh:/usr/src/app/docker/proxito.sh
       - ${PWD}:/usr/src/app/checkouts/readthedocs.org
       - ${PWD}/../readthedocs-ext:/usr/src/app/checkouts/readthedocs-ext
-      - ${PWD}/docker/settings/proxito.py:/usr/src/app/checkouts/readthedocs.org/readthedocs/settings/proxito.py
+      - ${PWD}/dockerfiles/settings/proxito.py:/usr/src/app/checkouts/readthedocs.org/readthedocs/settings/proxito.py
     links:
       - storage
       - database
@@ -55,12 +55,12 @@ services:
   web:
     image: readthedocsorg_server:latest
     volumes:
-      - ${PWD}/docker/scripts/createsuperuser.py:/usr/src/app/docker/createsuperuser.py
-      - ${PWD}/docker/entrypoints/common.sh:/usr/src/app/docker/common.sh
-      - ${PWD}/docker/entrypoints/web.sh:/usr/src/app/docker/web.sh
+      - ${PWD}/dockerfiles/scripts/createsuperuser.py:/usr/src/app/docker/createsuperuser.py
+      - ${PWD}/dockerfiles/entrypoints/common.sh:/usr/src/app/docker/common.sh
+      - ${PWD}/dockerfiles/entrypoints/web.sh:/usr/src/app/docker/web.sh
       - ${PWD}/../readthedocs-ext:/usr/src/app/checkouts/readthedocs-ext
       - ${PWD}:/usr/src/app/checkouts/readthedocs.org
-      - ${PWD}/docker/settings/web.py:/usr/src/app/checkouts/readthedocs.org/readthedocs/settings/web.py
+      - ${PWD}/dockerfiles/settings/web.py:/usr/src/app/checkouts/readthedocs.org/readthedocs/settings/web.py
     links:
       - storage
       - database
@@ -69,6 +69,7 @@ services:
       - storage
       - azure-cli
     environment:
+      - INIT=${INIT}
       - DJANGO_SETTINGS_MODULE=readthedocs.settings.web
     stdin_open: true
     tty: true
@@ -77,11 +78,12 @@ services:
   celery:
     image: readthedocsorg_server:latest
     volumes:
-      - ${PWD}/docker/entrypoints/common.sh:/usr/src/app/docker/common.sh
-      - ${PWD}/docker/entrypoints/celery.sh:/usr/src/app/docker/celery.sh
+      - ${PWD}/dockerfiles/entrypoints/common.sh:/usr/src/app/docker/common.sh
+      - ${PWD}/dockerfiles/entrypoints/celery.sh:/usr/src/app/docker/celery.sh
+      - ${PWD}/dockerfiles/scripts/wait_for_search.py:/usr/src/app/docker/scripts/wait_for_search.py
       - ${PWD}/../readthedocs-ext:/usr/src/app/checkouts/readthedocs-ext
       - ${PWD}:/usr/src/app/checkouts/readthedocs.org
-      - ${PWD}/docker/settings/celery.py:/usr/src/app/checkouts/readthedocs.org/readthedocs/settings/celery.py
+      - ${PWD}/dockerfiles/settings/celery.py:/usr/src/app/checkouts/readthedocs.org/readthedocs/settings/celery.py
     links:
       - storage
       - database
@@ -98,11 +100,11 @@ services:
   build:
     image: readthedocsorg_server:latest
     volumes:
-      - ${PWD}/docker/entrypoints/common.sh:/usr/src/app/docker/common.sh
-      - ${PWD}/docker/entrypoints/build.sh:/usr/src/app/docker/build.sh
+      - ${PWD}/dockerfiles/entrypoints/common.sh:/usr/src/app/docker/common.sh
+      - ${PWD}/dockerfiles/entrypoints/build.sh:/usr/src/app/docker/build.sh
       - ${PWD}/../readthedocs-ext:/usr/src/app/checkouts/readthedocs-ext
       - ${PWD}:/usr/src/app/checkouts/readthedocs.org
-      - ${PWD}/docker/settings/build.py:/usr/src/app/checkouts/readthedocs.org/readthedocs/settings/build.py
+      - ${PWD}/dockerfiles/settings/build.py:/usr/src/app/checkouts/readthedocs.org/readthedocs/settings/build.py
 
       # The python code at readthedocs/doc_builder/environments.py
       # mounts `self.project.doc_path`. We need to share this path
@@ -134,7 +136,7 @@ services:
   azure-cli:
     image: microsoft/azure-cli
     volumes:
-      - ${PWD}/docker/scripts/create-containers.sh:/usr/local/bin/create-containers.sh
+      - ${PWD}/dockerfiles/scripts/create-containers.sh:/usr/local/bin/create-containers.sh
     links:
       - storage
     depends_on:

docker/entrypoints/web.sh

@@ -1,7 +1,9 @@
 FROM ubuntu:18.04
 
 ENV DEBIAN_FRONTEND noninteractive
-ENV LANG C.UTF-8
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8
 
 WORKDIR /usr/src/app
 
@@ -14,6 +16,7 @@ RUN apt-get -y install \
         libpq-dev \
         libxml2-dev \
         libxslt1-dev \
+        locales \
         build-essential \
         python3-pip \
         python3-dev \
@@ -23,6 +26,10 @@ RUN apt-get -y install \
         libjpeg-dev \
         sqlite
 
+# Uncomment en_US.UTF-8 locale and generate it
+RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \
+    locale-gen
+
 RUN pip3 install --no-cache-dir --upgrade pip
 
 RUN mkdir checkouts

docker/Dockerfile renamed to dockerfiles/Dockerfile

@@ -2,6 +2,8 @@
 
 ../../docker/common.sh
 
+python3 ../../docker/scripts/wait_for_search.py
+
 watchmedo auto-restart \
   --patterns="*.py" \
   --ignore-patterns="*.#*.py;./user_builds/*;./public_*;./private_*;*.pyo;*.pyc;*flycheck*.py;./media/*;./.tox/*" \

docker/README.rst renamed to dockerfiles/README.rst

@@ -0,0 +1,14 @@
+#! /bin/sh
+
+../../docker/common.sh
+
+if [ -n "$INIT" ];
+then
+    echo "Performing initial tasks..."
+    python3 manage.py migrate
+    cat ../../docker/createsuperuser.py | python3 manage.py shell
+    python3 manage.py collectstatic --no-input
+    python3 manage.py loaddata test_data
+fi
+
+python3 manage.py runserver 0.0.0.0:8000

docker/entrypoints/build.sh renamed to dockerfiles/entrypoints/build.sh

@@ -0,0 +1,20 @@
+import os
+import time
+import requests
+
+wait_sleep = 3
+max_loops = 15
+
+loops = 1
+if 'SEARCH' in os.environ:
+    while loops < max_loops:
+        try:
+            print('Waiting for search service...')
+            response = requests.get('http://search:9200/_cluster/health')
+            if response.status_code == 200:
+                print('Search is ready!')
+                break
+            loops += 1
+        except:
+            pass
+        time.sleep(wait_sleep)

docker/entrypoints/celery.sh renamed to dockerfiles/entrypoints/celery.sh

@@ -0,0 +1,81 @@
+from invoke import task
+
+DOCKER_COMPOSE = 'docker-compose.yml'
+DOCKER_COMPOSE_SEARCH = 'docker-compose-search.yml'
+DOCKER_COMPOSE_COMMAND = f'docker-compose -f {DOCKER_COMPOSE} -f {DOCKER_COMPOSE_SEARCH}'
+
+@task
+def build(c):
+    """Build docker image for servers."""
+    c.run(f'{DOCKER_COMPOSE_COMMAND} build --no-cache', pty=True)
+
+@task
+def down(c, volumes=False):
+    """Stop and remove all the docker containers."""
+    if volumes:
+        c.run(f'{DOCKER_COMPOSE_COMMAND} down -v', pty=True)
+    else:
+        c.run(f'{DOCKER_COMPOSE_COMMAND} down', pty=True)
+
+@task
+def up(c, no_search=False, init=False):
+    """Start all the docker containers for a Read the Docs instance"""
+    INIT  = ''
+    if init:
+        INIT = 'INIT=t'
+
+    if no_search:
+        c.run(f'{INIT} docker-compose -f {DOCKER_COMPOSE} up', pty=True)
+    else:
+        c.run(f'{INIT} {DOCKER_COMPOSE_COMMAND} up', pty=True)
+
+@task
+def shell(c, running=False, container='web'):
+    """Run a shell inside a container."""
+    if running:
+        c.run(f'{DOCKER_COMPOSE_COMMAND} exec {container} /bin/bash', pty=True)
+    else:
+        c.run(f'{DOCKER_COMPOSE_COMMAND} run --rm {container} /bin/bash', pty=True)
+
+@task
+def manage(c, command):
+    """Run manage.py with a specific command."""
+    c.run(f'{DOCKER_COMPOSE_COMMAND} run --rm web python3 manage.py {command}', pty=True)
+
+@task
+def attach(c, container):
+    """Attach a tty to a running container (useful for pdb)."""
+    c.run(f'docker attach readthedocsorg_{container}_1', pty=True)
+
+@task
+def restart(c, containers):
+    """Restart one or more containers."""
+    c.run(f'{DOCKER_COMPOSE_COMMAND} restart {containers}', pty=True)
+
+    # When restarting a container that nginx is connected to, we need to restart
+    # nginx as well because it has the IP cached
+    need_nginx_restart = [
+        'web',
+        'proxito'
+        'storage',
+    ]
+    for extra in need_nginx_restart:
+        if extra in containers:
+            c.run(f'{DOCKER_COMPOSE_COMMAND} restart nginx', pty=True)
+            break
+
+@task
+def pull(c):
+    """Pull all docker images required for build servers."""
+    images = [
+        ('4.0', 'stable'),
+        ('5.0', 'latest'),
+    ]
+    for image, tag in images:
+        c.run(f'docker pull readthedocs/build:{image}', pty=True)
+        c.run(f'docker tag readthedocs/build:{image} readthedocs/build:{tag}', pty=True)
+
+@task
+def test(c, arguments=''):
+    """Run all test suite."""
+    c.run(f'{DOCKER_COMPOSE_COMMAND} run --rm --no-deps web tox {arguments}', pty=True)

docker/entrypoints/common.sh renamed to dockerfiles/entrypoints/common.sh

@@ -24,7 +24,7 @@ class DockerBaseSettings(CommunityDevSettings):
     MULTIPLE_BUILD_SERVERS = ['build']
 
     # Enable auto syncing elasticsearch documents
-    ELASTICSEARCH_DSL_AUTOSYNC = True
+    ELASTICSEARCH_DSL_AUTOSYNC = True if 'SEARCH' in os.environ else False
     ELASTICSEARCH_DSL = {
         'default': {
             'hosts': 'search:9200',

docker/entrypoints/proxito.sh renamed to dockerfiles/entrypoints/proxito.sh

@@ -11,3 +11,6 @@ watchdog==0.9.0
 datadiff==2.0.0
 ipdb==0.12.2
 pdbpp==0.10.2
+
+# run tests
+tox==3.14.1

dockerfiles/entrypoints/web.sh

@@ -7,6 +7,7 @@
 from invoke import task, Collection
 
 import common.tasks
+import dockerfiles.tasks
 
 
 ROOT_PATH = os.path.dirname(__file__)
@@ -35,6 +36,21 @@
     name='packages',
 )
 
+namespace.add_collection(
+    Collection(
+        dockerfiles.tasks.build,
+        dockerfiles.tasks.down,
+        dockerfiles.tasks.up,
+        dockerfiles.tasks.shell,
+        dockerfiles.tasks.manage,
+        dockerfiles.tasks.attach,
+        dockerfiles.tasks.restart,
+        dockerfiles.tasks.pull,
+        dockerfiles.tasks.test,
+    ),
+    name='docker',
+)
+
 # Localization tasks
 @task
 def push(ctx):