Docker compose setup

.dockerignore

@@ -0,0 +1,9 @@
+.git
+.gitignore
+.tox
+tox.ini
+.shipwright.json
+acceptance
+docs
+dockerfiles
+user_builds

.gitignore

@@ -5,6 +5,7 @@
 .DS_Store
 .coverage
 .idea
+.tox/
 .vagrant
 _build
 cnames

.shipwright.json

@@ -0,0 +1,11 @@
+{
+    "version": 1.0,
+    "namespace": "readthedocs",
+    "names": {
+        "/": "readthedocs/base",
+        "/dockerfiles/builder":  "readthedocs/builder",
+        "/dockerfiles/configs":  "readthedocs/configs",
+        "/dockerfiles/database": "readthedocs/database",
+        "/dockerfiles/webapp":   "readthedocs/webapp"
+    }
+}

Dockerfile

@@ -0,0 +1,32 @@
+#
+# readthedocs.org - base image
+#
+
+FROM    ubuntu:14.10
+MAINTAINER  [email protected]
+
+RUN     apt-get update && apt-get install -y \
+            python-dev \
+            python-pip \
+            python-virtualenv \
+            git \
+            build-essential \
+            libxml2-dev \
+            libxslt1-dev \
+            zlib1g-dev \
+            libpq-dev
+
+ADD     deploy_requirements.txt /rtd/deploy_requirements.txt
+ADD     pip_requirements.txt    /rtd/pip_requirements.txt
+ADD     deploy  /rtd/deploy
+WORKDIR /rtd
+RUN     virtualenv venv && venv/bin/pip install \
+            -r deploy_requirements.txt \
+            --find-links /rtd/deploy/wheels
+
+ADD     .   /rtd
+RUN     mkdir /rtd/user_builds
+RUN     mkdir /rtd/readthedocs/webapp_settings/
+
+ENV     PYTHONPATH  /rtd/
+WORKDIR /rtd/readthedocs

acceptance/environment.py

@@ -0,0 +1,4 @@
+
+
+def before_all(context):
+    context.base_url = 'http://localhost:8080'

acceptance/steps/websteps.py

@@ -0,0 +1,16 @@
+
+from pyquery import PyQuery as pq
+import requests
+
+
+@when(u'I view {url}')
+def step_impl(context, url):
+    context.response = requests.get(context.base_url + url)
+
+
+@then(u'I should see a list of projects')
+def step_impl(context):
+    assert context.response.status_code == 200
+    page = pq(context.response.text)
+    projects = page("li.module-item")
+    assert len(projects) == 14

acceptance/web.feature

@@ -0,0 +1,7 @@
+
+
+Feature: View projects
+
+  Scenario: View a list of projects
+    When I view /projects
+    Then I should see a list of projects

docker-compose.yml

@@ -0,0 +1,62 @@
+
+
+#
+# main application
+#
+webapp:
+    image: readthedocs/webapp
+    environment:
+        DJANGO_SETTINGS_MODULE: 'webapp_settings'
+    volumes_from:
+        - configs
+    volumes:
+        - './user_builds:/rtd/user_builds'
+    ports:
+        - 8080:8000
+    links:
+        - database
+        - redis
+        - search
+
+#
+# data volume container with a dev version of django settings
+#
+configs:
+    image: readthedocs/configs
+
+#
+# database for the webapp
+#
+database:
+    image: readthedocs/database
+
+#
+# task queue broker, and cache
+#
+redis:
+    image: redis:latest
+
+#
+# search index for documentation and projects
+#
+search:
+    image: readthedocs/search
+    ports: 
+        - 8081:9200
+
+#
+# task workers for building documentation
+#
+builder:
+    image: readthedocs/builder
+    environment:
+        DJANGO_SETTINGS_MODULE: 'webapp_settings'
+    volumes_from:
+        - configs
+    volumes:
+        - './user_builds:/rtd/user_builds'
+    links:
+        - redis
+        - webapp
+        - database # see github.com/rtfd/readthedocs.org/issues/987
+

dockerfiles/builder/Dockerfile

@@ -0,0 +1,17 @@
+#
+# readthedocs.org - documentation builder image
+#
+
+FROM    readthedocs/base
+
+# For some reason the task running runs virtualenv-2.7
+RUN     ln -s /usr/bin/virtualenv /usr/bin/virtualenv-2.7
+
+# Dependencies for building documentation
+RUN     apt-get update && apt-get install -y \
+            texlive-latex-base \
+            texlive-latex-extra \
+            texlive-latex-recommended \
+            texlive-fonts-recommended
+
+CMD     ../venv/bin/python manage.py celery worker -l INFO

dockerfiles/configs/Dockerfile

@@ -0,0 +1,8 @@
+#
+# readthedocs.org - configs image
+#
+
+FROM    busybox:latest
+ADD     . /rtd/readthedocs/webapp_settings/
+VOLUME  /rtd/readthedocs/webapp_settings/
+CMD     echo

dockerfiles/configs/__init__.py

@@ -0,0 +1,83 @@
+from readthedocs.settings.base import *  # noqa
+
+
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.postgresql_psycopg2',
+        'NAME': 'docs',
+        'USER': 'docs',
+        'PASSWORD': 'password',
+        'HOST': 'database_1',
+        'PORT': '5432',
+    }
+}
+
+
+LOG_FORMAT = "[%(asctime)s] %(levelname)s [%(name)s:%(lineno)s] %(message)s"
+LOGGING['loggers'] = {
+    '': {
+        'handlers': ['console'],
+        'level': 'DEBUG',
+    },
+}
+
+DEBUG = True
+TEMPLATE_DEBUG = False
+CELERY_ALWAYS_EAGER = False
+
+SESSION_ENGINE = "django.contrib.sessions.backends.cached_db"
+SESSION_COOKIE_DOMAIN = None
+SESSION_COOKIE_HTTPONLY = False
+
+CACHES = {
+    'default': {
+        'BACKEND': 'redis_cache.RedisCache',
+        'LOCATION': 'redis_1:6379',
+        'PREFIX': 'docs',
+        'OPTIONS': {
+            'DB': 1,
+            'PARSER_CLASS': 'redis.connection.HiredisParser'
+        },
+    },
+}
+
+
+REDIS = {
+    'host': 'redis_1',
+    'port': 6379,
+    'db': 0,
+}
+
+
+BROKER_URL = 'redis://redis_1:6379/0'
+CELERY_RESULT_BACKEND = 'redis://redis_1:6379/0'
+
+
+# Elasticsearch settings.
+ES_HOSTS = ['search_1:9200']
+
+
+SLUMBER_USERNAME = 'test'
+SLUMBER_PASSWORD = 'test'
+SLUMBER_API_HOST = 'http://webapp:8000'
+WEBSOCKET_HOST = 'localhost:8088'
+PRODUCTION_DOMAIN = 'localhost:8080'
+
+USE_SUBDOMAIN = False
+NGINX_X_ACCEL_REDIRECT = True
+
+SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
+
+# Lock builds for 10 minutes
+REPO_LOCK_SECONDS = 300
+
+# Don't re-confirm existing accounts
+ACCOUNT_EMAIL_VERIFICATION = 'none'
+
+# For testing locally.
+CORS_ORIGIN_WHITELIST = (
+    'localhost:8080',
+    'webapp_1:8000',
+)
+
+

dockerfiles/database/Dockerfile

@@ -0,0 +1,36 @@
+#
+# readthedocs.org - database image
+#
+
+FROM    readthedocs/base
+
+RUN     apt-get update && apt-get install -y \
+            postgresql-9.4 \
+            postgresql-contrib-9.4
+
+ADD     local_settings.py /rtd/readthedocs/local_settings.py
+ADD     create_superuser.py /rtd/readthedocs/create_superuser.py
+
+USER    postgres
+
+# See https://github.com/rtfd/readthedocs.org/issues/773
+# for the reason behind `migrate projects 0001` line
+RUN     /etc/init.d/postgresql start && \
+        psql --command "CREATE USER docs WITH SUPERUSER PASSWORD 'password';" && \
+        createdb docs && \
+        ../venv/bin/python ./manage.py syncdb --noinput && \
+        ../venv/bin/python ./manage.py migrate projects 0001 && \
+        ../venv/bin/python ./manage.py migrate && \
+        ../venv/bin/python ./create_superuser.py && \
+        ../venv/bin/python ./manage.py loaddata test_data && \
+        /etc/init.d/postgresql stop
+
+ADD     etc/pg_hba.conf /etc/postgresql/9.4/main/pg_hba.conf
+RUN     echo "listen_addresses='*'" >> /etc/postgresql/9.4/main/postgresql.conf
+
+# Add VOLUMEs to allow backup of config, logs and databases
+VOLUME  ["/etc/postgresql", "/var/log/postgresql", "/var/lib/postgresql"]
+
+EXPOSE  5432
+CMD ["/usr/lib/postgresql/9.4/bin/postgres", "-D", "/var/lib/postgresql/9.4/main", "-c", "config_file=/etc/postgresql/9.4/main/postgresql.conf"]
+

dockerfiles/database/create_superuser.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+import os
+import sys
+
+
+if __name__ == "__main__":
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings.sqlite")
+    sys.path.append('readthedocs')
+
+    from django.contrib.auth.models import User
+    User.objects.create_superuser('admin', '[email protected]', 'password')

dockerfiles/database/etc/pg_hba.conf

@@ -0,0 +1,2 @@
+# TYPE  DATABASE        USER            ADDRESS                 METHOD
+host    all             all             0.0.0.0/0               md5

dockerfiles/database/local_settings.py

@@ -0,0 +1,38 @@
+
+
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.postgresql_psycopg2',
+        'NAME': 'docs',
+        'USER': 'docs',
+        'PASSWORD': 'password',
+        'HOST': 'localhost',
+        'PORT': '5432',
+    }
+}
+
+LOG_FORMAT = "[%(asctime)s] %(levelname)s [%(name)s:%(lineno)s] %(message)s"
+LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': True,
+    'formatters': {
+        'standard': {
+            'format': LOG_FORMAT,
+            'datefmt': "%d/%b/%Y %H:%M:%S"
+        },
+    },
+    'handlers': {
+        'console': {
+            'level': 'INFO',
+            'class': 'logging.StreamHandler',
+            'formatter': 'standard'
+        },
+    },
+    'loggers': {
+        '': {
+            'handlers': ['console'],
+            'level': 'INFO',
+        },
+    }
+}
+

dockerfiles/webapp/Dockerfile

@@ -0,0 +1,10 @@
+#
+# readthedocs.org - webapp
+#
+
+FROM    readthedocs/base
+
+
+# TODO: use gunicorn
+EXPOSE  8000
+CMD     ../venv/bin/python ./manage.py runserver 0.0.0.0:8000

readthedocs/projects/tasks.py

@@ -302,6 +302,8 @@ def setup_environment(version):
         site_packages = '--system-site-packages'
     else:
         site_packages = '--no-site-packages'
+
+    # TODO: why is this virtualenv-2.7?
     # Here the command has been modified to support different
     # interpreters.
     ret_dict['venv'] = run(