Restructure of Docker command execution and large build refactor

docs/development/buildenvironments.rst

@@ -0,0 +1,68 @@
+==================
+Build Environments
+==================
+
+Read the Docs uses container virtualization to encapsulate documentation build
+processes. Each build spins up a new virtual machinge using our base image,
+which is an image with the minimum necessary components required to build
+documentation. Virtual machines are limiting in CPU time and memory, which aims
+to reduce excessive usage of build resources.
+
+Setup
+-----
+
+Build environments use `Docker`_ to handle container virtualization. To perform
+any development on the Docker build system, you will need to set up `Docker`_ on
+your host system. Setup of Docker will vary by system, and so is out of the
+scope of this documentation.
+
+Once you have Docker set up, you will need to create the base image used for
+container creation. The ``base`` image is found in our `container images repo`_.
+It is the basic container image supported by our community site.
+
+To get started, create the image using the `docker` command line tool. You can
+name the image whatever you like here, ``rtfd-build`` is the default name, but
+can be configured in your settings -- see `Configuration`_::
+
+    docker build -t rtfd-build base/
+
+When this process has completed, you should have a working image that Read the
+Docs can use to start containers.
+
+.. _`Docker`: http://docker.com
+.. _`container images repo`: https://github.com/rtfd/readthedocs-docker-images
+
+Configuration
+-------------
+
+There are several settings used to configure usage of virtual machines:
+
+    DOCKER_ENABLED
+        True/False value used to enable the Docker build environment. Default:
+        False
+
+    DOCKER_LIMITS
+        A dictionary of limits to virtual machines. These limits include:
+
+            time
+                An integer representing the total allowed time limit (in
+                seconds) of build processes. This time limit affects the parent
+                process to the virtual machine and will force a virtual machine
+                to die if a build is still running after the alloted time
+                expires.
+
+            memory
+                The maximum memory allocated to the virtual machine. If this
+                limit is hit, build processes will be automatically killed.
+                Examples: '200m' for 200MB of total memory, or '2g' for 2GB of
+                total memory.
+
+    DOCKER_IMAGE
+        Tag of a Docker image to use as a base image.
+
+    DOCKER_SOCKET
+        URI of the socket to connect to the Docker daemon. Examples include:
+        ``unix:///var/run/docker.sock`` and ``tcp://127.0.0.1:2375``
+
+    DOCKER_VERSION
+        Version of the API to use for the Docker API client.

docs/index.rst

@@ -77,7 +77,8 @@ Information about development is also available:
    contribute
    tests
    architecture
-   development
+   development/standards
+   development/buildenvironments
    symlinks
    settings
    i18n

readthedocs/builds/constants.py

@@ -1,11 +1,17 @@
 from django.utils.translation import ugettext_lazy as _
 
+BUILD_STATE_TRIGGERED = 'triggered'
+BUILD_STATE_CLONING = 'cloning'
+BUILD_STATE_INSTALLING = 'installing'
+BUILD_STATE_BUILDING = 'building'
+BUILD_STATE_FINISHED = 'finished'
+
 BUILD_STATE = (
-    ('triggered', _('Triggered')),
-    ('cloning', _('Cloning')),
-    ('installing', _('Installing')),
-    ('building', _('Building')),
-    ('finished', _('Finished')),
+    (BUILD_STATE_TRIGGERED, _('Triggered')),
+    (BUILD_STATE_CLONING, _('Cloning')),
+    (BUILD_STATE_INSTALLING, _('Installing')),
+    (BUILD_STATE_BUILDING, _('Building')),
+    (BUILD_STATE_FINISHED, _('Finished')),
 )
 
 BUILD_TYPES = (

readthedocs/core/management/commands/update_api.py

@@ -14,10 +14,14 @@ class Command(BaseCommand):
     the site. Invoked via ``./manage.py update_repos``.
     """
 
+    def add_arguments(self, parser):
+        parser.add_argument('--docker', action='store_true', default=False)
+
     def handle(self, *args, **options):
+        docker = options.get('docker', False)
         if len(args):
             for slug in args:
                 project_data = api.project(slug).get()
                 p = tasks.make_api_project(project_data)
                 log.info("Building %s" % p)
-                tasks.update_docs(pk=p.pk)
+                tasks.update_docs.run(pk=p.pk, docker=docker)

readthedocs/core/management/commands/update_repos.py

@@ -48,9 +48,9 @@ def handle(self, *args, **options):
                     for version in Version.objects.filter(project__slug=slug,
                                                           active=True,
                                                           uploaded=False):
-                        tasks.update_docs(pk=version.project_id,
-                                          record=False,
-                                          version_pk=version.pk)
+                        tasks.update_docs.run(pk=version.project_id,
+                                              record=False,
+                                              version_pk=version.pk)
                 else:
                     p = Project.all_objects.get(slug=slug)
                     log.info("Building %s" % p)
@@ -60,14 +60,15 @@ def handle(self, *args, **options):
                 log.info("Updating all versions")
                 for version in Version.objects.filter(active=True,
                                                       uploaded=False):
-                    tasks.update_docs(pk=version.project_id,
-                                      record=record,
-                                      force=force,
-                                      version_pk=version.pk)
+                    tasks.update_docs.run(pk=version.project_id,
+                                          record=record,
+                                          force=force,
+                                          version_pk=version.pk)
             else:
                 log.info("Updating all docs")
-                tasks.update_docs_pull(record=record,
-                                       force=force)
+                for project in Project.objects.all():
+                    tasks.update_docs.run(pk=project.pk, record=record,
+                                          force=force)
 
     @property
     def help(self):

readthedocs/core/management/commands/update_versions.py

@@ -1,5 +1,6 @@
-from readthedocs.builds.models import Version
 from django.core.management.base import BaseCommand
+
+from readthedocs.builds.models import Version
 from readthedocs.projects.tasks import update_docs
 
 
@@ -10,5 +11,5 @@ class Command(BaseCommand):
 
     def handle(self, *args, **options):
         for version in Version.objects.filter(active=True, built=False):
-            update_docs(version.project_id, record=False,
-                        version_pk=version.pk)
+            update_docs.run(version.project_id, record=False,
+                            version_pk=version.pk)

readthedocs/doc_builder/backends/mkdocs.py

@@ -7,7 +7,6 @@
 from django.template import Context, loader as template_loader
 
 from readthedocs.doc_builder.base import BaseBuilder, restoring_chdir
-from readthedocs.projects.utils import run
 
 log = logging.getLogger(__name__)
 
@@ -20,6 +19,7 @@ class BaseMkdocs(BaseBuilder):
     """
     Mkdocs builder
     """
+    use_theme = True
 
     def __init__(self, *args, **kwargs):
         super(BaseMkdocs, self).__init__(*args, **kwargs)
@@ -127,21 +127,18 @@ def append_conf(self, **kwargs):
         include_file.write(include_string)
         include_file.close()
 
-    @restoring_chdir
     def build(self, **kwargs):
-        checkout_path = self.version.project.checkout_path(self.version.slug)
-        # site_path = os.path.join(checkout_path, 'site')
-        os.chdir(checkout_path)
-        # Actual build
-        build_command = (
-            "{command} {builder} --clean --site-dir={build_dir} --theme=readthedocs"
-            .format(
-                command=self.version.project.venv_bin(version=self.version.slug, bin='mkdocs'),
-                builder=self.builder,
-                build_dir=self.build_dir,
-            ))
-        results = run(build_command, shell=True)
-        return results
+        checkout_path = self.project.checkout_path(self.version.slug)
+        build_command = [
+            self.project.venv_bin(version=self.version.slug, bin='mkdocs'),
+            self.builder,
+            '--clean',
+            '--site-dir', self.build_dir,
+        ]
+        if self.use_theme:
+            build_command.extend(['--theme', 'readthedocs'])
+        cmd_ret = self.run(*build_command, cwd=checkout_path)
+        return cmd_ret.successful
 
 
 class MkdocsHTML(BaseMkdocs):
@@ -154,3 +151,4 @@ class MkdocsJSON(BaseMkdocs):
     type = 'mkdocs_json'
     builder = 'json'
     build_dir = '_build/json'
+    use_theme = False