From ac97f07235c8af427e961b9abf184d180dc78f51 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 16 Oct 2014 00:12:06 -0400 Subject: [PATCH 1/2] Version 1 of a docker compose demo --- .dockerignore | 4 ++ .gitignore | 1 + Dockerfile | 37 +++++++++++ acceptance/builds.feature | 7 ++ acceptance/environment.py | 4 ++ acceptance/steps/websteps.py | 16 +++++ dockerfiles/builder/Dockerfile | 17 +++++ dockerfiles/configs/Dockerfile | 6 ++ dockerfiles/configs/__init__.py | 83 ++++++++++++++++++++++++ dockerfiles/database/Dockerfile | 39 +++++++++++ dockerfiles/database/create_superuser.py | 11 ++++ dockerfiles/database/etc/pg_hba.conf | 2 + dockerfiles/database/local_settings.py | 38 +++++++++++ fig.yml | 62 ++++++++++++++++++ readthedocs/projects/tasks.py | 2 + readthedocs/search/Dockerfile | 26 ++++++++ readthedocs/search/create_index.py | 26 ++++++++ readthedocs/search/elasticsearch.yml | 8 +++ readthedocs/search/indexes.py | 20 ++++-- readthedocs/search/settings.py | 6 ++ tox.ini | 32 +++++++++ 21 files changed, 440 insertions(+), 7 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 acceptance/builds.feature create mode 100644 acceptance/environment.py create mode 100644 acceptance/steps/websteps.py create mode 100644 dockerfiles/builder/Dockerfile create mode 100644 dockerfiles/configs/Dockerfile create mode 100644 dockerfiles/configs/__init__.py create mode 100644 dockerfiles/database/Dockerfile create mode 100644 dockerfiles/database/create_superuser.py create mode 100644 dockerfiles/database/etc/pg_hba.conf create mode 100644 dockerfiles/database/local_settings.py create mode 100644 fig.yml create mode 100644 readthedocs/search/Dockerfile create mode 100644 readthedocs/search/create_index.py create mode 100644 readthedocs/search/elasticsearch.yml create mode 100644 readthedocs/search/settings.py create mode 100644 tox.ini diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000000..92384136b6f --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +.git +.tox +dockerfiles +user_builds diff --git a/.gitignore b/.gitignore index 9e41f5aa5ca..7a309ce407f 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ .DS_Store .coverage .idea +.tox/ .vagrant _build cnames diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000000..bb11d99c46d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,37 @@ +# +# readthedocs.org - webapp +# + +FROM ubuntu:14.10 +MAINTAINER dnephin@gmail.com + +RUN apt-get update && apt-get install -y \ + python-dev \ + python-pip \ + python-virtualenv \ + git \ + build-essential \ + libxml2-dev \ + libxslt1-dev \ + zlib1g-dev \ + libpq-dev + +ADD deploy_requirements.txt /rtd/deploy_requirements.txt +ADD pip_requirements.txt /rtd/pip_requirements.txt +ADD deploy /rtd/deploy +WORKDIR /rtd +RUN virtualenv venv && venv/bin/pip install \ + -r deploy_requirements.txt \ + --find-links /rtd/deploy/wheels + +ADD . /rtd +RUN mkdir /rtd/user_builds +RUN mkdir /rtd/readthedocs/webapp_settings/ + +EXPOSE 8000 + +ENV PYTHONPATH /rtd/ +WORKDIR /rtd/readthedocs + +# TODO: use gunicorn +CMD ../venv/bin/python ./manage.py runserver 0.0.0.0:8000 diff --git a/acceptance/builds.feature b/acceptance/builds.feature new file mode 100644 index 00000000000..fae390f1906 --- /dev/null +++ b/acceptance/builds.feature @@ -0,0 +1,7 @@ + + +Feature: Build documentation for projects + + Scenario: View a list of builds for a project + When I view /builds/pip + Then I should see a list of builds diff --git a/acceptance/environment.py b/acceptance/environment.py new file mode 100644 index 00000000000..503fb58ebaa --- /dev/null +++ b/acceptance/environment.py @@ -0,0 +1,4 @@ + + +def before_all(context): + context.base_url = 'http://localhost:8080' diff --git a/acceptance/steps/websteps.py b/acceptance/steps/websteps.py new file mode 100644 index 00000000000..9670f2536f3 --- /dev/null +++ b/acceptance/steps/websteps.py @@ -0,0 +1,16 @@ + +from pyquery import PyQuery as pq +import requests + + +@when(u'I view {url}') +def step_impl(context, url): + context.response = requests.get(context.base_url + url) + + +@then(u'I should see a list of builds') +def step_impl(context): + assert context.response.status_code == 200 + page = pq(context.response.text) + builds = page("li.module-item div[id^=build-]") + assert len(builds) > 1 diff --git a/dockerfiles/builder/Dockerfile b/dockerfiles/builder/Dockerfile new file mode 100644 index 00000000000..14d7991b26c --- /dev/null +++ b/dockerfiles/builder/Dockerfile @@ -0,0 +1,17 @@ + + +# TODO: It would be nice to have a separate code repo for the task workers, or +# a common base image for python dependencies +FROM rtd_webapp + +# For some reason the task running runs virtualenv-2.7 +RUN ln -s /usr/bin/virtualenv /usr/bin/virtualenv-2.7 + +# Dependencies for building documentation +RUN apt-get update && apt-get install -y \ + texlive-latex-base \ + texlive-latex-extra \ + texlive-latex-recommended \ + texlive-fonts-recommended + +CMD ../venv/bin/python manage.py celery worker -l INFO diff --git a/dockerfiles/configs/Dockerfile b/dockerfiles/configs/Dockerfile new file mode 100644 index 00000000000..627ff1f3b40 --- /dev/null +++ b/dockerfiles/configs/Dockerfile @@ -0,0 +1,6 @@ + + +FROM busybox:latest +ADD . /rtd/readthedocs/webapp_settings/ +VOLUME /rtd/readthedocs/webapp_settings/ +CMD echo diff --git a/dockerfiles/configs/__init__.py b/dockerfiles/configs/__init__.py new file mode 100644 index 00000000000..909d1f7ca0b --- /dev/null +++ b/dockerfiles/configs/__init__.py @@ -0,0 +1,83 @@ +from readthedocs.settings.base import * # noqa + + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.postgresql_psycopg2', + 'NAME': 'docs', + 'USER': 'docs', + 'PASSWORD': 'password', + 'HOST': 'database_1', + 'PORT': '5432', + } +} + + +LOG_FORMAT = "[%(asctime)s] %(levelname)s [%(name)s:%(lineno)s] %(message)s" +LOGGING['loggers'] = { + '': { + 'handlers': ['console'], + 'level': 'DEBUG', + }, +} + +DEBUG = True +TEMPLATE_DEBUG = False +CELERY_ALWAYS_EAGER = False + +SESSION_ENGINE = "django.contrib.sessions.backends.cached_db" +SESSION_COOKIE_DOMAIN = None +SESSION_COOKIE_HTTPONLY = False + +CACHES = { + 'default': { + 'BACKEND': 'redis_cache.RedisCache', + 'LOCATION': 'redis_1:6379', + 'PREFIX': 'docs', + 'OPTIONS': { + 'DB': 1, + 'PARSER_CLASS': 'redis.connection.HiredisParser' + }, + }, +} + + +REDIS = { + 'host': 'redis_1', + 'port': 6379, + 'db': 0, +} + + +BROKER_URL = 'redis://redis_1:6379/0' +CELERY_RESULT_BACKEND = 'redis://redis_1:6379/0' + + +# Elasticsearch settings. +ES_HOSTS = ['search_1:9200'] + + +SLUMBER_USERNAME = 'test' +SLUMBER_PASSWORD = 'test' +SLUMBER_API_HOST = 'http://webapp:8000' +WEBSOCKET_HOST = 'localhost:8088' +PRODUCTION_DOMAIN = 'localhost:8080' + +USE_SUBDOMAIN = False +NGINX_X_ACCEL_REDIRECT = True + +SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") + +# Lock builds for 10 minutes +REPO_LOCK_SECONDS = 300 + +# Don't re-confirm existing accounts +ACCOUNT_EMAIL_VERIFICATION = 'none' + +# For testing locally. +CORS_ORIGIN_WHITELIST = ( + 'localhost:8080', + 'webapp_1:8000', +) + + diff --git a/dockerfiles/database/Dockerfile b/dockerfiles/database/Dockerfile new file mode 100644 index 00000000000..b5b9e3cecaa --- /dev/null +++ b/dockerfiles/database/Dockerfile @@ -0,0 +1,39 @@ +# +# readthedocs.org - database +# + +# TODO: It would be nice to not have to install all the python stuff +# just to run ./manage.py syncdb/migration +FROM rtd_webapp +MAINTAINER dnephin@gmail.com + +RUN apt-get update && apt-get install -y \ + postgresql-9.3 \ + postgresql-contrib-9.3 + +ADD local_settings.py /rtd/readthedocs/local_settings.py +ADD create_superuser.py /rtd/readthedocs/create_superuser.py + +USER postgres + +# See https://github.com/rtfd/readthedocs.org/issues/773 +# for the reason behind `migrate projects 0001` line +RUN /etc/init.d/postgresql start && \ + psql --command "CREATE USER docs WITH SUPERUSER PASSWORD 'password';" && \ + createdb docs && \ + ../venv/bin/python ./manage.py syncdb --noinput && \ + ../venv/bin/python ./manage.py migrate projects 0001 && \ + ../venv/bin/python ./manage.py migrate && \ + ../venv/bin/python ./create_superuser.py && \ + ../venv/bin/python ./manage.py loaddata test_data && \ + /etc/init.d/postgresql stop + +ADD etc/pg_hba.conf /etc/postgresql/9.3/main/pg_hba.conf +RUN echo "listen_addresses='*'" >> /etc/postgresql/9.3/main/postgresql.conf + +# Add VOLUMEs to allow backup of config, logs and databases +VOLUME ["/etc/postgresql", "/var/log/postgresql", "/var/lib/postgresql"] + +EXPOSE 5432 +CMD ["/usr/lib/postgresql/9.3/bin/postgres", "-D", "/var/lib/postgresql/9.3/main", "-c", "config_file=/etc/postgresql/9.3/main/postgresql.conf"] + diff --git a/dockerfiles/database/create_superuser.py b/dockerfiles/database/create_superuser.py new file mode 100644 index 00000000000..c89f132434f --- /dev/null +++ b/dockerfiles/database/create_superuser.py @@ -0,0 +1,11 @@ +#!/usr/bin/env python +import os +import sys + + +if __name__ == "__main__": + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings.sqlite") + sys.path.append('readthedocs') + + from django.contrib.auth.models import User + User.objects.create_superuser('admin', 'admin@example.com', 'password') diff --git a/dockerfiles/database/etc/pg_hba.conf b/dockerfiles/database/etc/pg_hba.conf new file mode 100644 index 00000000000..af536826b86 --- /dev/null +++ b/dockerfiles/database/etc/pg_hba.conf @@ -0,0 +1,2 @@ +# TYPE DATABASE USER ADDRESS METHOD +host all all 0.0.0.0/0 md5 diff --git a/dockerfiles/database/local_settings.py b/dockerfiles/database/local_settings.py new file mode 100644 index 00000000000..afd87d2bca3 --- /dev/null +++ b/dockerfiles/database/local_settings.py @@ -0,0 +1,38 @@ + + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.postgresql_psycopg2', + 'NAME': 'docs', + 'USER': 'docs', + 'PASSWORD': 'password', + 'HOST': 'localhost', + 'PORT': '5432', + } +} + +LOG_FORMAT = "[%(asctime)s] %(levelname)s [%(name)s:%(lineno)s] %(message)s" +LOGGING = { + 'version': 1, + 'disable_existing_loggers': True, + 'formatters': { + 'standard': { + 'format': LOG_FORMAT, + 'datefmt': "%d/%b/%Y %H:%M:%S" + }, + }, + 'handlers': { + 'console': { + 'level': 'INFO', + 'class': 'logging.StreamHandler', + 'formatter': 'standard' + }, + }, + 'loggers': { + '': { + 'handlers': ['console'], + 'level': 'INFO', + }, + } +} + diff --git a/fig.yml b/fig.yml new file mode 100644 index 00000000000..600268b6a08 --- /dev/null +++ b/fig.yml @@ -0,0 +1,62 @@ + + +# +# main application +# +webapp: + build: . + environment: + DJANGO_SETTINGS_MODULE: 'webapp_settings' + volumes_from: + - configs + volumes: + - './user_builds:/rtd/user_builds' + ports: + - 8080:8000 + links: + - database + - redis + - search + +# +# data volume container with a dev version of django settings +# +configs: + build: dockerfiles/configs + +# +# database for the webapp +# +database: + build: dockerfiles/database + +# +# task queue broker, and cache +# +redis: + image: redis:latest + +# +# search index for documentation and projects +# +search: + build: readthedocs/search + ports: + - 8081:9200 + +# +# task workers for building documentation +# +builder: + build: dockerfiles/builder + environment: + DJANGO_SETTINGS_MODULE: 'webapp_settings' + volumes_from: + - configs + volumes: + - './user_builds:/rtd/user_builds' + links: + - redis + - webapp + - database # see github.com/rtfd/readthedocs.org/issues/987 + diff --git a/readthedocs/projects/tasks.py b/readthedocs/projects/tasks.py index 97b3e188de2..0737a4afc44 100644 --- a/readthedocs/projects/tasks.py +++ b/readthedocs/projects/tasks.py @@ -302,6 +302,8 @@ def setup_environment(version): site_packages = '--system-site-packages' else: site_packages = '--no-site-packages' + + # TODO: why is this virtualenv-2.7? # Here the command has been modified to support different # interpreters. ret_dict['venv'] = run( diff --git a/readthedocs/search/Dockerfile b/readthedocs/search/Dockerfile new file mode 100644 index 00000000000..c612ac05f6e --- /dev/null +++ b/readthedocs/search/Dockerfile @@ -0,0 +1,26 @@ + + +FROM dockerfile/elasticsearch + +RUN apt-get update && apt-get install -y \ + python-virtualenv \ + python-pip + +WORKDIR /search +RUN virtualenv .venv && \ + .venv/bin/pip install elasticsearch==0.4.3 + +ADD elasticsearch.yml /elasticsearch/config/elasticsearch.yml + +RUN /elasticsearch/bin/plugin -install \ + elasticsearch/elasticsearch-analysis-icu/2.3.0 && \ + /elasticsearch/bin/plugin -install mobz/elasticsearch-head + + +ADD . /search + +# TODO: sleep could be replaced with more of a polling wait for startup +RUN /elasticsearch/bin/elasticsearch & \ + sleep 8 && \ + curl -s localhost:9200/_nodes/_all/plugins | python -m json.tool && \ + .venv/bin/python create_index.py diff --git a/readthedocs/search/create_index.py b/readthedocs/search/create_index.py new file mode 100644 index 00000000000..c6851082b77 --- /dev/null +++ b/readthedocs/search/create_index.py @@ -0,0 +1,26 @@ +import logging + +from indexes import ( + Index, + PageIndex, + ProjectIndex, + SectionIndex, +) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + + # Create the index. + index = Index() + index_name = index.timestamped_index() + index.create_index(index_name) + index.update_aliases(index_name) + + # Update mapping + proj = ProjectIndex() + proj.put_mapping() + page = PageIndex() + page.put_mapping() + sec = SectionIndex() + sec.put_mapping() diff --git a/readthedocs/search/elasticsearch.yml b/readthedocs/search/elasticsearch.yml new file mode 100644 index 00000000000..684cb1831f8 --- /dev/null +++ b/readthedocs/search/elasticsearch.yml @@ -0,0 +1,8 @@ + +path: + # TODO: /data is being cleared after steps + # maybe because of VOLUME # ["/data/"] + data: /searchindex + logs: /data/logs + plugins: /elasticsearch/plugins + work: /data/work diff --git a/readthedocs/search/indexes.py b/readthedocs/search/indexes.py index 7a445296734..2b394d924ac 100644 --- a/readthedocs/search/indexes.py +++ b/readthedocs/search/indexes.py @@ -19,7 +19,12 @@ from elasticsearch import Elasticsearch, exceptions from elasticsearch.helpers import bulk_index -from django.conf import settings +# TODO: fix this, it's here to deal with different execution environments +# django webapp, vs building fixtures for the search docker container +try: + from django.conf import settings +except ImportError: + import settings class Index(object): @@ -318,13 +323,14 @@ def get_mapping(self): '_boost': {'name': '_boost', 'null_value': 1.0}, # Associate a section with a page. '_parent': {'type': self._parent}, - 'suggest': { - "type": "completion", - "index_analyzer": "simple", - "search_analyzer": "simple", - "payloads": True, - }, 'properties': { + # FIXME: this moved to support ES 1.2.x + 'suggest': { + "type": "completion", + "index_analyzer": "simple", + "search_analyzer": "simple", + "payloads": True, + }, 'id': {'type': 'string', 'index': 'not_analyzed'}, 'project': {'type': 'string', 'index': 'not_analyzed'}, 'version': {'type': 'string', 'index': 'not_analyzed'}, diff --git a/readthedocs/search/settings.py b/readthedocs/search/settings.py new file mode 100644 index 00000000000..06a241f484e --- /dev/null +++ b/readthedocs/search/settings.py @@ -0,0 +1,6 @@ + +# Elasticsearch settings. +ES_HOSTS = ['localhost:9200'] +ES_DEFAULT_NUM_REPLICAS = 1 +ES_DEFAULT_NUM_SHARDS = 1 + diff --git a/tox.ini b/tox.ini new file mode 100644 index 00000000000..578b35d7763 --- /dev/null +++ b/tox.ini @@ -0,0 +1,32 @@ +[tox] +envlist = py27,acceptance +skipsdist = true + +[testenv] +usedevelop = True +deps = + -rpip_requirements.txt + pytest + coverage +commands = + ./runtests.sh {posargs} + + +[testenv:acceptance] +setenv = + FIG_PROJECT_NAME = rtd +deps = + fig + behave + requests + pyquery +commands = + # Remove old data volume so we can pickup any changes + fig rm --force configs + # Build all the images + fig build + # Start the containers + fig up -d + # Run the tests + behave acceptance + fig stop From 595d01cee993ffe3e1d668753f8ccc29858e6ad5 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Sun, 25 Jan 2015 16:19:52 -0500 Subject: [PATCH 2/2] Use shipwright and docker-compose. --- .dockerignore | 5 +++++ .shipwright.json | 11 +++++++++++ Dockerfile | 7 +------ acceptance/builds.feature | 7 ------- acceptance/steps/websteps.py | 6 +++--- acceptance/web.feature | 7 +++++++ fig.yml => docker-compose.yml | 10 +++++----- dockerfiles/builder/Dockerfile | 8 ++++---- dockerfiles/configs/Dockerfile | 4 +++- dockerfiles/database/Dockerfile | 17 +++++++---------- dockerfiles/webapp/Dockerfile | 10 ++++++++++ readthedocs/search/Dockerfile | 2 +- tox.ini | 20 ++++++++++++++------ 13 files changed, 71 insertions(+), 43 deletions(-) create mode 100644 .shipwright.json delete mode 100644 acceptance/builds.feature create mode 100644 acceptance/web.feature rename fig.yml => docker-compose.yml (85%) create mode 100644 dockerfiles/webapp/Dockerfile diff --git a/.dockerignore b/.dockerignore index 92384136b6f..9240b2a41b2 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,9 @@ .git +.gitignore .tox +tox.ini +.shipwright.json +acceptance +docs dockerfiles user_builds diff --git a/.shipwright.json b/.shipwright.json new file mode 100644 index 00000000000..edc34c6b1e5 --- /dev/null +++ b/.shipwright.json @@ -0,0 +1,11 @@ +{ + "version": 1.0, + "namespace": "readthedocs", + "names": { + "/": "readthedocs/base", + "/dockerfiles/builder": "readthedocs/builder", + "/dockerfiles/configs": "readthedocs/configs", + "/dockerfiles/database": "readthedocs/database", + "/dockerfiles/webapp": "readthedocs/webapp" + } +} diff --git a/Dockerfile b/Dockerfile index bb11d99c46d..dfb9bfc6238 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # -# readthedocs.org - webapp +# readthedocs.org - base image # FROM ubuntu:14.10 @@ -28,10 +28,5 @@ ADD . /rtd RUN mkdir /rtd/user_builds RUN mkdir /rtd/readthedocs/webapp_settings/ -EXPOSE 8000 - ENV PYTHONPATH /rtd/ WORKDIR /rtd/readthedocs - -# TODO: use gunicorn -CMD ../venv/bin/python ./manage.py runserver 0.0.0.0:8000 diff --git a/acceptance/builds.feature b/acceptance/builds.feature deleted file mode 100644 index fae390f1906..00000000000 --- a/acceptance/builds.feature +++ /dev/null @@ -1,7 +0,0 @@ - - -Feature: Build documentation for projects - - Scenario: View a list of builds for a project - When I view /builds/pip - Then I should see a list of builds diff --git a/acceptance/steps/websteps.py b/acceptance/steps/websteps.py index 9670f2536f3..40a6eeb44dc 100644 --- a/acceptance/steps/websteps.py +++ b/acceptance/steps/websteps.py @@ -8,9 +8,9 @@ def step_impl(context, url): context.response = requests.get(context.base_url + url) -@then(u'I should see a list of builds') +@then(u'I should see a list of projects') def step_impl(context): assert context.response.status_code == 200 page = pq(context.response.text) - builds = page("li.module-item div[id^=build-]") - assert len(builds) > 1 + projects = page("li.module-item") + assert len(projects) == 14 diff --git a/acceptance/web.feature b/acceptance/web.feature new file mode 100644 index 00000000000..3479490a334 --- /dev/null +++ b/acceptance/web.feature @@ -0,0 +1,7 @@ + + +Feature: View projects + + Scenario: View a list of projects + When I view /projects + Then I should see a list of projects diff --git a/fig.yml b/docker-compose.yml similarity index 85% rename from fig.yml rename to docker-compose.yml index 600268b6a08..1fb39404a85 100644 --- a/fig.yml +++ b/docker-compose.yml @@ -4,7 +4,7 @@ # main application # webapp: - build: . + image: readthedocs/webapp environment: DJANGO_SETTINGS_MODULE: 'webapp_settings' volumes_from: @@ -22,13 +22,13 @@ webapp: # data volume container with a dev version of django settings # configs: - build: dockerfiles/configs + image: readthedocs/configs # # database for the webapp # database: - build: dockerfiles/database + image: readthedocs/database # # task queue broker, and cache @@ -40,7 +40,7 @@ redis: # search index for documentation and projects # search: - build: readthedocs/search + image: readthedocs/search ports: - 8081:9200 @@ -48,7 +48,7 @@ search: # task workers for building documentation # builder: - build: dockerfiles/builder + image: readthedocs/builder environment: DJANGO_SETTINGS_MODULE: 'webapp_settings' volumes_from: diff --git a/dockerfiles/builder/Dockerfile b/dockerfiles/builder/Dockerfile index 14d7991b26c..0c5d6e3587b 100644 --- a/dockerfiles/builder/Dockerfile +++ b/dockerfiles/builder/Dockerfile @@ -1,8 +1,8 @@ +# +# readthedocs.org - documentation builder image +# - -# TODO: It would be nice to have a separate code repo for the task workers, or -# a common base image for python dependencies -FROM rtd_webapp +FROM readthedocs/base # For some reason the task running runs virtualenv-2.7 RUN ln -s /usr/bin/virtualenv /usr/bin/virtualenv-2.7 diff --git a/dockerfiles/configs/Dockerfile b/dockerfiles/configs/Dockerfile index 627ff1f3b40..ab2b47465b0 100644 --- a/dockerfiles/configs/Dockerfile +++ b/dockerfiles/configs/Dockerfile @@ -1,4 +1,6 @@ - +# +# readthedocs.org - configs image +# FROM busybox:latest ADD . /rtd/readthedocs/webapp_settings/ diff --git a/dockerfiles/database/Dockerfile b/dockerfiles/database/Dockerfile index b5b9e3cecaa..bb6f03fb66a 100644 --- a/dockerfiles/database/Dockerfile +++ b/dockerfiles/database/Dockerfile @@ -1,15 +1,12 @@ # -# readthedocs.org - database +# readthedocs.org - database image # -# TODO: It would be nice to not have to install all the python stuff -# just to run ./manage.py syncdb/migration -FROM rtd_webapp -MAINTAINER dnephin@gmail.com +FROM readthedocs/base RUN apt-get update && apt-get install -y \ - postgresql-9.3 \ - postgresql-contrib-9.3 + postgresql-9.4 \ + postgresql-contrib-9.4 ADD local_settings.py /rtd/readthedocs/local_settings.py ADD create_superuser.py /rtd/readthedocs/create_superuser.py @@ -28,12 +25,12 @@ RUN /etc/init.d/postgresql start && \ ../venv/bin/python ./manage.py loaddata test_data && \ /etc/init.d/postgresql stop -ADD etc/pg_hba.conf /etc/postgresql/9.3/main/pg_hba.conf -RUN echo "listen_addresses='*'" >> /etc/postgresql/9.3/main/postgresql.conf +ADD etc/pg_hba.conf /etc/postgresql/9.4/main/pg_hba.conf +RUN echo "listen_addresses='*'" >> /etc/postgresql/9.4/main/postgresql.conf # Add VOLUMEs to allow backup of config, logs and databases VOLUME ["/etc/postgresql", "/var/log/postgresql", "/var/lib/postgresql"] EXPOSE 5432 -CMD ["/usr/lib/postgresql/9.3/bin/postgres", "-D", "/var/lib/postgresql/9.3/main", "-c", "config_file=/etc/postgresql/9.3/main/postgresql.conf"] +CMD ["/usr/lib/postgresql/9.4/bin/postgres", "-D", "/var/lib/postgresql/9.4/main", "-c", "config_file=/etc/postgresql/9.4/main/postgresql.conf"] diff --git a/dockerfiles/webapp/Dockerfile b/dockerfiles/webapp/Dockerfile new file mode 100644 index 00000000000..6c03c278920 --- /dev/null +++ b/dockerfiles/webapp/Dockerfile @@ -0,0 +1,10 @@ +# +# readthedocs.org - webapp +# + +FROM readthedocs/base + + +# TODO: use gunicorn +EXPOSE 8000 +CMD ../venv/bin/python ./manage.py runserver 0.0.0.0:8000 diff --git a/readthedocs/search/Dockerfile b/readthedocs/search/Dockerfile index c612ac05f6e..df3364d23c3 100644 --- a/readthedocs/search/Dockerfile +++ b/readthedocs/search/Dockerfile @@ -1,6 +1,6 @@ -FROM dockerfile/elasticsearch +FROM dockerfile/elasticsearch:latest RUN apt-get update && apt-get install -y \ python-virtualenv \ diff --git a/tox.ini b/tox.ini index 578b35d7763..dcba06b47a7 100644 --- a/tox.ini +++ b/tox.ini @@ -13,20 +13,28 @@ commands = [testenv:acceptance] +whitelist_externals = + sleep setenv = - FIG_PROJECT_NAME = rtd + COMPOSE_PROJECT_NAME = rtd deps = - fig + docker-py == 0.6.0 + websocket-client == 0.11 + shipwright + docker-compose behave requests pyquery commands = # Remove old data volume so we can pickup any changes - fig rm --force configs + docker-compose rm --force configs database # Build all the images - fig build + shipwright # Start the containers - fig up -d + docker-compose up -d + # Wait for everything to start + sleep 5 # Run the tests behave acceptance - fig stop + # Shutdown + docker-compose stop