Build process: use Celery handlers

.circleci/config.yml

@@ -10,6 +10,7 @@ jobs:
         name: search
         environment:
           discovery.type: single-node
+          ES_JAVA_OPTS: -Xms750m -Xmx750m
     steps:
       - checkout
       - run: git submodule sync

docs/user/faq.rst

@@ -122,12 +122,6 @@ following settings::
     SLUMBER_PASSWORD = 'test'
 
 
-Deleting a stale or broken build environment
---------------------------------------------
-
-See :doc:`guides/wipe-environment`.
-
-
 How do I host multiple projects on one custom domain?
 -----------------------------------------------------
 

docs/user/guides/administrators.rst

@@ -19,4 +19,3 @@ have a look at our :doc:`/tutorial/index`.
    deprecating-content
    pdf-non-ascii-languages
    importing-private-repositories
-   wipe-environment

readthedocs/api/v3/tests/test_builds.py

@@ -5,7 +5,7 @@
 from .mixins import APIEndpointMixin
 
 
-@mock.patch('readthedocs.projects.tasks.update_docs_task', mock.MagicMock())
+@mock.patch('readthedocs.projects.tasks.builds.update_docs_task', mock.MagicMock())
 class BuildsEndpointTests(APIEndpointMixin):
 
     def test_projects_builds_list(self):

readthedocs/api/v3/tests/test_projects.py

@@ -8,7 +8,7 @@
 from .mixins import APIEndpointMixin
 
 
-@mock.patch('readthedocs.projects.tasks.update_docs_task', mock.MagicMock())
+@mock.patch('readthedocs.projects.tasks.builds.update_docs_task', mock.MagicMock())
 class ProjectsEndpointTests(APIEndpointMixin):
 
     def test_projects_list(self):

readthedocs/builds/admin.py

@@ -20,7 +20,6 @@
     VersionAutomationRule,
 )
 from readthedocs.core.utils import trigger_build
-from readthedocs.core.utils.general import wipe_version_via_slugs
 from readthedocs.projects.models import HTMLFile
 from readthedocs.search.utils import _indexing_helper
 
@@ -118,29 +117,15 @@ class VersionAdmin(admin.ModelAdmin):
     list_filter = ('type', 'privacy_level', 'active', 'built')
     search_fields = ('slug', 'project__slug')
     raw_id_fields = ('project',)
-    actions = ['build_version', 'reindex_version', 'wipe_version', 'wipe_selected_versions']
+    actions = ['build_version', 'reindex_version', 'wipe_version_indexes']
 
     def project_slug(self, obj):
         return obj.project.slug
 
-    def wipe_selected_versions(self, request, queryset):
-        """Wipes the selected versions."""
-        for version in queryset:
-            wipe_version_via_slugs(
-                version_slug=version.slug,
-                project_slug=version.project.slug
-            )
-            self.message_user(
-                request,
-                'Wiped {}.'.format(version.slug),
-                level=messages.SUCCESS
-            )
-
     def pretty_config(self, instance):
         return _pretty_config(instance)
 
     pretty_config.short_description = 'Config File'
-    wipe_selected_versions.short_description = 'Wipe selected versions'
 
     def build_version(self, request, queryset):
         """Trigger a build for the project version."""
@@ -179,7 +164,7 @@ def reindex_version(self, request, queryset):
 
     reindex_version.short_description = 'Reindex version to ES'
 
-    def wipe_version(self, request, queryset):
+    def wipe_version_indexes(self, request, queryset):
         """Wipe selected versions from ES."""
         html_objs_qs = []
         for version in queryset.iterator():
@@ -197,7 +182,7 @@ def wipe_version(self, request, queryset):
             messages.SUCCESS,
         )
 
-    wipe_version.short_description = 'Wipe version from ES'
+    wipe_version_indexes.short_description = 'Wipe version from ES'
 
 
 @admin.register(RegexAutomationRule)

readthedocs/builds/models.py

@@ -317,9 +317,9 @@ def get_absolute_url(self):
         )
 
     def delete(self, *args, **kwargs):  # pylint: disable=arguments-differ
-        from readthedocs.projects import tasks
+        from readthedocs.projects.tasks.utils import clean_project_resources
         log.info('Removing files for version.', version_slug=self.slug)
-        tasks.clean_project_resources(self.project, self)
+        clean_project_resources(self.project, self)
         super().delete(*args, **kwargs)
 
     @property
@@ -614,6 +614,9 @@ class Build(models.Model):
     date = models.DateTimeField(_('Date'), auto_now_add=True, db_index=True)
     success = models.BooleanField(_('Success'), default=True)
 
+    # TODO: remove these fields (setup, setup_error, output, error, exit_code)
+    # since they are not used anymore in the new implementation and only really
+    # old builds (>5 years ago) only were using these fields.
     setup = models.TextField(_('Setup'), null=True, blank=True)
     setup_error = models.TextField(_('Setup error'), null=True, blank=True)
     output = models.TextField(_('Output'), default='', blank=True)

readthedocs/builds/tasks.py

@@ -68,8 +68,8 @@ class TaskRouter:
     def route_for_task(self, task, args, kwargs, **__):
         log.debug('Executing TaskRouter.', task=task)
         if task not in (
-            'readthedocs.projects.tasks.update_docs_task',
-            'readthedocs.projects.tasks.sync_repository_task',
+            'readthedocs.projects.tasks.builds.update_docs_task',
+            'readthedocs.projects.tasks.builds.sync_repository_task',
         ):
             log.debug('Skipping routing non-build task.', task=task)
             return
@@ -153,8 +153,8 @@ def route_for_task(self, task, args, kwargs, **__):
 
     def _get_version(self, task, args, kwargs):
         tasks = [
-            'readthedocs.projects.tasks.update_docs_task',
-            'readthedocs.projects.tasks.sync_repository_task',
+            'readthedocs.projects.tasks.builds.update_docs_task',
+            'readthedocs.projects.tasks.builds.sync_repository_task',
         ]
         version = None
         if task in tasks:

readthedocs/builds/tests/test_celery_task_router.py

@@ -29,7 +29,7 @@ def setUp(self):
 
             )
 
-        self.task = 'readthedocs.projects.tasks.update_docs_task'
+        self.task = 'readthedocs.projects.tasks.builds.update_docs_task'
         self.args = (
             self.version.pk,
         )

readthedocs/builds/views.py

@@ -18,7 +18,7 @@
 from readthedocs.builds.models import Build, Version
 from readthedocs.core.permissions import AdminPermission
 from readthedocs.core.utils import trigger_build
-from readthedocs.doc_builder.exceptions import BuildEnvironmentError
+from readthedocs.doc_builder.exceptions import BuildAppError
 from readthedocs.projects.models import Project
 
 log = structlog.get_logger(__name__)
@@ -154,7 +154,7 @@ def get_context_data(self, **kwargs):
 
         build = self.get_object()
 
-        if build.error != BuildEnvironmentError.GENERIC_WITH_BUILD_ID.format(build_id=build.pk):
+        if build.error != BuildAppError.GENERIC_WITH_BUILD_ID.format(build_id=build.pk):
             # Do not suggest to open an issue if the error is not generic
             return context
 

readthedocs/config/config.py

@@ -72,6 +72,7 @@
 LATEST_CONFIGURATION_VERSION = 2
 
 
+# TODO: make these exception to inherit from `BuildUserError`
 class ConfigError(Exception):
 
     """Base error for the rtd configuration file."""

readthedocs/core/management/commands/update_repos.py

@@ -25,14 +25,6 @@ class Command(BaseCommand):
     def add_arguments(self, parser):
         parser.add_argument('--slugs', nargs='+', type=str)
 
-        parser.add_argument(
-            '-f',
-            action='store_true',
-            dest='force',
-            default=False,
-            help='Force a build in sphinx',
-        )
-
         parser.add_argument(
             '-V',
             dest='version',
@@ -41,7 +33,6 @@ def add_arguments(self, parser):
         )
 
     def handle(self, *args, **options):
-        force = options['force']
         version = options['version']
         slugs = options.get('slugs', [])
 
@@ -85,7 +76,7 @@ def handle(self, *args, **options):
                 else:
                     p = Project.all_objects.get(slug=slug)
                     log.info('Building ...', project_slug=p.slug)
-                    trigger_build(project=p, force=force)
+                    trigger_build(project=p)
         else:
             if version == 'all':
                 log.info('Updating all versions')

readthedocs/core/tests/test_history.py

@@ -1,3 +1,5 @@
+from unittest import mock
+
 from django.contrib.auth.models import User
 from django.test import TestCase
 from django.urls import reverse
@@ -13,6 +15,7 @@ def setUp(self):
         self.project = get(Project, users=[self.user])
         self.client.force_login(self.user)
 
+    @mock.patch('readthedocs.projects.forms.trigger_build', mock.MagicMock())
     def test_extra_historical_fields_with_request(self):
         self.assertEqual(self.project.history.count(), 1)
         r = self.client.post(