diff --git a/common b/common index bc413ae5338..4982ec21605 160000 --- a/common +++ b/common @@ -1 +1 @@ -Subproject commit bc413ae53380dbb9902c75c6c22ea5b67b1b0d2b +Subproject commit 4982ec21605ad5e6c481c104943c46b50725c93e diff --git a/docker-compose.yml b/docker-compose.yml index b59b220bc68..54bf0cd691d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,8 +33,8 @@ services: proxito: image: readthedocsorg_server:latest volumes: - - ${PWD}/dockerfiles/entrypoints/common.sh:/usr/src/app/docker/common.sh - - ${PWD}/dockerfiles/entrypoints/proxito.sh:/usr/src/app/docker/proxito.sh + - ${PWD}/common/dockerfiles/entrypoints/common.sh:/usr/src/app/docker/common.sh + - ${PWD}/common/dockerfiles/entrypoints/proxito.sh:/usr/src/app/docker/proxito.sh - ${PWD}:/usr/src/app/checkouts/readthedocs.org - ${PWD}/../readthedocs-ext:/usr/src/app/checkouts/readthedocs-ext - ${PWD}/dockerfiles/settings/proxito.py:/usr/src/app/checkouts/readthedocs.org/readthedocs/settings/proxito.py @@ -56,9 +56,9 @@ services: web: image: readthedocsorg_server:latest volumes: - - ${PWD}/dockerfiles/scripts/createsuperuser.py:/usr/src/app/docker/createsuperuser.py - - ${PWD}/dockerfiles/entrypoints/common.sh:/usr/src/app/docker/common.sh - - ${PWD}/dockerfiles/entrypoints/web.sh:/usr/src/app/docker/web.sh + - ${PWD}/common/dockerfiles/scripts/createsuperuser.py:/usr/src/app/docker/createsuperuser.py + - ${PWD}/common/dockerfiles/entrypoints/common.sh:/usr/src/app/docker/common.sh + - ${PWD}/common/dockerfiles/entrypoints/web.sh:/usr/src/app/docker/web.sh - ${PWD}/../readthedocs-ext:/usr/src/app/checkouts/readthedocs-ext - ${PWD}:/usr/src/app/checkouts/readthedocs.org - ${PWD}/dockerfiles/settings/web.py:/usr/src/app/checkouts/readthedocs.org/readthedocs/settings/web.py @@ -80,9 +80,9 @@ services: celery: image: readthedocsorg_server:latest volumes: - - ${PWD}/dockerfiles/entrypoints/common.sh:/usr/src/app/docker/common.sh - - ${PWD}/dockerfiles/entrypoints/celery.sh:/usr/src/app/docker/celery.sh - - ${PWD}/dockerfiles/scripts/wait_for_search.py:/usr/src/app/docker/scripts/wait_for_search.py + - ${PWD}/common/dockerfiles/entrypoints/common.sh:/usr/src/app/docker/common.sh + - ${PWD}/common/dockerfiles/entrypoints/celery.sh:/usr/src/app/docker/celery.sh + - ${PWD}/common/dockerfiles/scripts/wait_for_search.py:/usr/src/app/docker/scripts/wait_for_search.py - ${PWD}/../readthedocs-ext:/usr/src/app/checkouts/readthedocs-ext - ${PWD}:/usr/src/app/checkouts/readthedocs.org - ${PWD}/dockerfiles/settings/celery.py:/usr/src/app/checkouts/readthedocs.org/readthedocs/settings/celery.py @@ -103,8 +103,8 @@ services: build: image: readthedocsorg_server:latest volumes: - - ${PWD}/dockerfiles/entrypoints/common.sh:/usr/src/app/docker/common.sh - - ${PWD}/dockerfiles/entrypoints/build.sh:/usr/src/app/docker/build.sh + - ${PWD}/common/dockerfiles/entrypoints/common.sh:/usr/src/app/docker/common.sh + - ${PWD}/common/dockerfiles/entrypoints/build.sh:/usr/src/app/docker/build.sh - ${PWD}/../readthedocs-ext:/usr/src/app/checkouts/readthedocs-ext - ${PWD}:/usr/src/app/checkouts/readthedocs.org - ${PWD}/dockerfiles/settings/build.py:/usr/src/app/checkouts/readthedocs.org/readthedocs/settings/build.py @@ -140,7 +140,7 @@ services: azure-cli: image: microsoft/azure-cli volumes: - - ${PWD}/dockerfiles/scripts/create-containers.sh:/usr/local/bin/create-containers.sh + - ${PWD}/common/dockerfiles/scripts/create-containers.sh:/usr/local/bin/create-containers.sh links: - storage depends_on: diff --git a/dockerfiles/entrypoints/build.sh b/dockerfiles/entrypoints/build.sh deleted file mode 100755 index a32146210b1..00000000000 --- a/dockerfiles/entrypoints/build.sh +++ /dev/null @@ -1,23 +0,0 @@ -#! /bin/bash - -../../docker/common.sh - -CMD='python3 -m celery worker -A readthedocs.worker -Ofair -c 2 -Q builder,celery,default,build01 -l DEBUG' - -if [ -n "${DOCKER_NO_RELOAD}" ]; then - echo "Running Docker with no reload" - $CMD -else - echo "Running Docker with reload" - watchmedo auto-restart \ - --patterns="*.py" \ - --ignore-patterns="*.#*.py;./user_builds/*;./public_*;./private_*;*.pyo;*.pyc;*flycheck*.py;./media/*;./.tox/*" \ - --ignore-directories \ - --recursive \ - --signal=SIGTERM \ - --kill-after=5 \ - --interval=5 \ - -- \ - $CMD -fi - diff --git a/dockerfiles/entrypoints/celery.sh b/dockerfiles/entrypoints/celery.sh deleted file mode 100755 index d9b5e166e92..00000000000 --- a/dockerfiles/entrypoints/celery.sh +++ /dev/null @@ -1,24 +0,0 @@ -#! /bin/sh - -../../docker/common.sh - -python3 ../../docker/scripts/wait_for_search.py - -CMD='python3 -m celery worker -A readthedocs.worker -Ofair -c 2 -Q web,web01,reindex -l DEBUG' - -if [ -n "${DOCKER_NO_RELOAD}" ]; then - echo "Running Docker with no reload" - $CMD -else - echo "Running Docker with reload" - watchmedo auto-restart \ - --patterns="*.py" \ - --ignore-patterns="*.#*.py;./user_builds/*;./public_*;./private_*;*.pyo;*.pyc;*flycheck*.py;./media/*;./.tox/*" \ - --ignore-directories \ - --recursive \ - --signal=SIGTERM \ - --kill-after=5 \ - --interval=5 \ - -- \ - $CMD -fi diff --git a/dockerfiles/entrypoints/common.sh b/dockerfiles/entrypoints/common.sh deleted file mode 100755 index a678eeb0351..00000000000 --- a/dockerfiles/entrypoints/common.sh +++ /dev/null @@ -1,7 +0,0 @@ -#! /bin/sh - -# This file is useful when you need to run a command on all the Python -# instances and you don't want to rebuild the image. For example, when -# working on a feature that requires a new dependency and you need to -# call: "pip install newdependency==1.4.2". You can add that command -# here, and it will be executed in all the instances at startup. diff --git a/dockerfiles/entrypoints/proxito.sh b/dockerfiles/entrypoints/proxito.sh deleted file mode 100755 index b6a2776a087..00000000000 --- a/dockerfiles/entrypoints/proxito.sh +++ /dev/null @@ -1,14 +0,0 @@ -#! /bin/sh - -../../docker/common.sh - -if [ -n "${DOCKER_NO_RELOAD}" ]; -then - RELOAD='--noreload' - echo "Running Docker with no reload" -else - RELOAD='' - echo "Running Docker with reload" -fi - -python3 manage.py runserver 0.0.0.0:8000 $RELOAD diff --git a/dockerfiles/entrypoints/web.sh b/dockerfiles/entrypoints/web.sh deleted file mode 100755 index 3589898fcc6..00000000000 --- a/dockerfiles/entrypoints/web.sh +++ /dev/null @@ -1,23 +0,0 @@ -#! /bin/sh - -../../docker/common.sh - -if [ -n "$INIT" ]; -then - echo "Performing initial tasks..." - python3 manage.py migrate - cat ../../docker/createsuperuser.py | python3 manage.py shell - python3 manage.py collectstatic --no-input - python3 manage.py loaddata test_data -fi - -if [ -n "${DOCKER_NO_RELOAD}" ]; -then - RELOAD='--noreload' - echo "Running Docker with no reload" -else - RELOAD='' - echo "Running Docker with reload" -fi - -python3 manage.py runserver 0.0.0.0:8000 $RELOAD diff --git a/dockerfiles/scripts/create-containers.sh b/dockerfiles/scripts/create-containers.sh deleted file mode 100755 index 0fd0ad9def8..00000000000 --- a/dockerfiles/scripts/create-containers.sh +++ /dev/null @@ -1,5 +0,0 @@ -#! /bin/sh - -az storage container create --connection-string "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://storage:10000/devstoreaccount1" --public-access "blob" --name "builds" -az storage container create --connection-string "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://storage:10000/devstoreaccount1" --public-access "blob" --name "media" -az storage container create --connection-string "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://storage:10000/devstoreaccount1" --public-access "blob" --name "static" diff --git a/dockerfiles/scripts/createsuperuser.py b/dockerfiles/scripts/createsuperuser.py deleted file mode 100644 index 66aa4d5b77a..00000000000 --- a/dockerfiles/scripts/createsuperuser.py +++ /dev/null @@ -1,4 +0,0 @@ -from django.contrib.auth.models import User - -if not User.objects.filter(username='admin').exists(): - User.objects.create_superuser('admin', 'admin@admin.com', 'admin') diff --git a/dockerfiles/scripts/wait_for_search.py b/dockerfiles/scripts/wait_for_search.py deleted file mode 100644 index 28e9d1f1d9b..00000000000 --- a/dockerfiles/scripts/wait_for_search.py +++ /dev/null @@ -1,20 +0,0 @@ -import os -import time -import requests - -wait_sleep = 3 -max_loops = 15 - -loops = 1 -if 'SEARCH' in os.environ: - while loops < max_loops: - try: - print('Waiting for search service...') - response = requests.get('http://search:9200/_cluster/health') - if response.status_code == 200: - print('Search is ready!') - break - loops += 1 - except: - pass - time.sleep(wait_sleep) diff --git a/dockerfiles/tasks.py b/dockerfiles/tasks.py deleted file mode 100644 index b1e0c16c8ae..00000000000 --- a/dockerfiles/tasks.py +++ /dev/null @@ -1,86 +0,0 @@ -from invoke import task - -DOCKER_COMPOSE = 'docker-compose.yml' -DOCKER_COMPOSE_SEARCH = 'docker-compose-search.yml' -DOCKER_COMPOSE_COMMAND = f'docker-compose -f {DOCKER_COMPOSE} -f {DOCKER_COMPOSE_SEARCH}' - -@task -def build(c): - """Build docker image for servers.""" - c.run(f'{DOCKER_COMPOSE_COMMAND} build --no-cache', pty=True) - -@task -def down(c, volumes=False): - """Stop and remove all the docker containers.""" - if volumes: - c.run(f'{DOCKER_COMPOSE_COMMAND} down -v', pty=True) - else: - c.run(f'{DOCKER_COMPOSE_COMMAND} down', pty=True) - - -@task -def up(c, no_search=False, init=False, no_reload=False): - """Start all the docker containers for a Read the Docs instance""" - INIT = 'INIT=' - DOCKER_NO_RELOAD = 'DOCKER_NO_RELOAD=' - if init: - INIT = 'INIT=t' - if no_reload: - DOCKER_NO_RELOAD = 'DOCKER_NO_RELOAD=t' - - if no_search: - c.run(f'{INIT} {DOCKER_NO_RELOAD} docker-compose -f {DOCKER_COMPOSE} up', pty=True) - else: - c.run(f'{INIT} {DOCKER_NO_RELOAD} {DOCKER_COMPOSE_COMMAND} up', pty=True) - - -@task -def shell(c, running=False, container='web'): - """Run a shell inside a container.""" - if running: - c.run(f'{DOCKER_COMPOSE_COMMAND} exec {container} /bin/bash', pty=True) - else: - c.run(f'{DOCKER_COMPOSE_COMMAND} run --rm {container} /bin/bash', pty=True) - -@task -def manage(c, command): - """Run manage.py with a specific command.""" - c.run(f'{DOCKER_COMPOSE_COMMAND} run --rm web python3 manage.py {command}', pty=True) - -@task -def attach(c, container): - """Attach a tty to a running container (useful for pdb).""" - c.run(f'docker attach readthedocsorg_{container}_1', pty=True) - -@task -def restart(c, containers): - """Restart one or more containers.""" - c.run(f'{DOCKER_COMPOSE_COMMAND} restart {containers}', pty=True) - - # When restarting a container that nginx is connected to, we need to restart - # nginx as well because it has the IP cached - need_nginx_restart = [ - 'web', - 'proxito' - 'storage', - ] - for extra in need_nginx_restart: - if extra in containers: - c.run(f'{DOCKER_COMPOSE_COMMAND} restart nginx', pty=True) - break - -@task -def pull(c): - """Pull all docker images required for build servers.""" - images = [ - ('4.0', 'stable'), - ('5.0', 'latest'), - ] - for image, tag in images: - c.run(f'docker pull readthedocs/build:{image}', pty=True) - c.run(f'docker tag readthedocs/build:{image} readthedocs/build:{tag}', pty=True) - -@task -def test(c, arguments=''): - """Run all test suite.""" - c.run(f'{DOCKER_COMPOSE_COMMAND} run --rm --no-deps web tox {arguments}', pty=True) diff --git a/tasks.py b/tasks.py index 95b3a4075f9..f7c17322607 100644 --- a/tasks.py +++ b/tasks.py @@ -1,13 +1,11 @@ """Read the Docs tasks.""" -from __future__ import division, print_function, unicode_literals - import os from invoke import task, Collection import common.tasks -import dockerfiles.tasks +import common.dockerfiles.tasks ROOT_PATH = os.path.dirname(__file__) @@ -37,16 +35,11 @@ ) namespace.add_collection( - Collection( - dockerfiles.tasks.build, - dockerfiles.tasks.down, - dockerfiles.tasks.up, - dockerfiles.tasks.shell, - dockerfiles.tasks.manage, - dockerfiles.tasks.attach, - dockerfiles.tasks.restart, - dockerfiles.tasks.pull, - dockerfiles.tasks.test, + Collection.from_module( + common.dockerfiles.tasks, + config={ + 'container_prefix': 'readthedocsorg', + }, ), name='docker', )