Remove broadcast function

readthedocs/builds/models.py

@@ -60,7 +60,6 @@
 )
 from readthedocs.builds.version_slug import VersionSlugField
 from readthedocs.config import LATEST_CONFIGURATION_VERSION
-from readthedocs.core.utils import broadcast
 from readthedocs.projects.constants import (
     BITBUCKET_COMMIT_URL,
     BITBUCKET_URL,

readthedocs/core/utils/__init__.py

@@ -5,62 +5,28 @@
 import os
 import re
 
-from celery import chord, group
 from django.conf import settings
 from django.utils.functional import keep_lazy
 from django.utils.safestring import SafeText, mark_safe
 from django.utils.text import slugify as slugify_base
 
 from readthedocs.builds.constants import (
-    BUILD_STATE_TRIGGERED,
     BUILD_STATE_FINISHED,
+    BUILD_STATE_TRIGGERED,
     BUILD_STATUS_PENDING,
     EXTERNAL,
 )
 from readthedocs.doc_builder.constants import DOCKER_LIMITS
-from readthedocs.projects.constants import CELERY_LOW, CELERY_MEDIUM, CELERY_HIGH
 from readthedocs.doc_builder.exceptions import BuildMaxConcurrencyError
-
+from readthedocs.projects.constants import (
+    CELERY_HIGH,
+    CELERY_LOW,
+    CELERY_MEDIUM,
+)
 
 log = logging.getLogger(__name__)
 
 
-def broadcast(type, task, args, kwargs=None, callback=None):  # pylint: disable=redefined-builtin
-    """
-    Run a broadcast across our servers.
-
-    Returns a task group that can be checked for results.
-
-    `callback` should be a task signature that will be run once,
-    after all of the broadcast tasks have finished running.
-    """
-    if type not in ['web', 'app', 'build']:
-        raise ValueError('allowed value of `type` are web, app and build.')
-    if kwargs is None:
-        kwargs = {}
-
-    if type in ['web', 'app']:
-        servers = settings.MULTIPLE_APP_SERVERS
-    elif type in ['build']:
-        servers = settings.MULTIPLE_BUILD_SERVERS
-
-    tasks = []
-    for server in servers:
-        task_sig = task.s(*args, **kwargs).set(queue=server)
-        tasks.append(task_sig)
-    if callback:
-        task_promise = chord(tasks, callback).apply_async()
-    else:
-        # Celery's Group class does some special handling when an iterable with
-        # len() == 1 is passed in. This will be hit if there is only one server
-        # defined in the above queue lists
-        if len(tasks) > 1:
-            task_promise = group(*tasks).apply_async()
-        else:
-            task_promise = group(tasks).apply_async()
-    return task_promise
-
-
 def prepare_build(
         project,
         version=None,

readthedocs/core/utils/general.py

@@ -1,13 +1,7 @@
-# -*- coding: utf-8 -*-
-
-import os
-
 from django.conf import settings
 from django.core.files.storage import get_storage_class
 from django.shortcuts import get_object_or_404
 
-from readthedocs.core.utils import broadcast
-from readthedocs.projects.tasks import remove_dirs
 from readthedocs.builds.models import Version
 
 
@@ -18,14 +12,6 @@ def wipe_version_via_slugs(version_slug, project_slug):
         slug=version_slug,
         project__slug=project_slug,
     )
-    del_dirs = [
-        os.path.join(version.project.doc_path, 'checkouts', version.slug),
-        os.path.join(version.project.doc_path, 'envs', version.slug),
-        os.path.join(version.project.doc_path, 'conda', version.slug),
-        os.path.join(version.project.doc_path, '.cache'),
-    ]
-    for del_dir in del_dirs:
-        broadcast(type='build', task=remove_dirs, args=[(del_dir,)])
 
     # Delete the cache environment from storage
     storage = get_storage_class(settings.RTD_BUILD_ENVIRONMENT_STORAGE)()

readthedocs/projects/admin.py

@@ -221,12 +221,7 @@ def ban_owner(self, request, queryset):
     ban_owner.short_description = 'Ban project owner'
 
     def delete_selected_and_artifacts(self, request, queryset):
-        """
-        Remove HTML/etc artifacts from storage.
-
-        Prior to the query delete, broadcast tasks to delete HTML artifacts from
-        application instances.
-        """
+        """Remove HTML/etc artifacts from storage."""
         if request.POST.get('post'):
             for project in queryset:
                 clean_project_resources(project)

readthedocs/projects/models.py

@@ -4,6 +4,7 @@
 import logging
 import os
 import re
+from shlex import quote
 from urllib.parse import urlparse
 
 from allauth.socialaccount.providers import registry as allauth_registry
@@ -16,42 +17,43 @@
 from django.utils.functional import cached_property
 from django.utils.translation import ugettext_lazy as _
 from django_extensions.db.models import TimeStampedModel
-from shlex import quote
 from taggit.managers import TaggableManager
 
 from readthedocs.api.v2.client import api
-from readthedocs.builds.constants import LATEST, STABLE, INTERNAL, EXTERNAL
+from readthedocs.builds.constants import EXTERNAL, INTERNAL, LATEST, STABLE
 from readthedocs.core.resolver import resolve, resolve_domain
-from readthedocs.core.utils import broadcast, slugify
+from readthedocs.core.utils import slugify
 from readthedocs.doc_builder.constants import DOCKER_LIMITS
 from readthedocs.projects import constants
 from readthedocs.projects.exceptions import ProjectConfigurationError
 from readthedocs.projects.managers import HTMLFileManager
 from readthedocs.projects.querysets import (
     ChildRelatedProjectQuerySet,
     FeatureQuerySet,
+    HTMLFileQuerySet,
     ProjectQuerySet,
     RelatedProjectQuerySet,
-    HTMLFileQuerySet,
 )
 from readthedocs.projects.templatetags.projects_tags import sort_version_aware
 from readthedocs.projects.validators import (
     validate_domain_name,
     validate_repository_url,
 )
 from readthedocs.projects.version_handling import determine_stable_version
-from readthedocs.search.parse_json import process_file, process_mkdocs_index_file
+from readthedocs.search.parse_json import (
+    process_file,
+    process_mkdocs_index_file,
+)
 from readthedocs.vcs_support.backends import backend_cls
 from readthedocs.vcs_support.utils import Lock, NonBlockingLock
 
 from .constants import (
-    MEDIA_TYPES,
-    MEDIA_TYPE_PDF,
     MEDIA_TYPE_EPUB,
     MEDIA_TYPE_HTMLZIP,
+    MEDIA_TYPE_PDF,
+    MEDIA_TYPES,
 )
 
-
 log = logging.getLogger(__name__)
 DOC_PATH_PREFIX = getattr(settings, 'DOC_PATH_PREFIX', '')
 

readthedocs/projects/tasks.py

@@ -86,7 +86,6 @@
     files_changed,
 )
 
-
 log = logging.getLogger(__name__)
 
 
@@ -1091,12 +1090,7 @@ def update_app_instances(
             pdf=False,
             epub=False,
     ):
-        """
-        Update application instances with build artifacts.
-
-        This triggers updates across application instances for html, pdf, epub,
-        downloads, and search. Tasks are broadcast to all web servers from here.
-        """
+        """Update build artifacts and index search data."""
         # Update version if we have successfully built HTML output
         # And store whether the build had other media types
         try:
@@ -1114,11 +1108,8 @@ def update_app_instances(
                 'Updating version failed, skipping file sync: version=%s',
                 self.version,
             )
-        hostname = socket.gethostname()
-
-        delete_unsynced_media = True
 
-        # Broadcast finalization steps to web application instances
+        # Index search data
         fileify.delay(
             version_pk=self.version.pk,
             commit=self.build['commit'],

readthedocs/projects/views/private.py

@@ -44,7 +44,7 @@
     LoginRequiredMixin,
     PrivateViewMixin,
 )
-from readthedocs.core.utils import broadcast, trigger_build
+from readthedocs.core.utils import trigger_build
 from readthedocs.core.utils.extend import SettingsOverrideObject
 from readthedocs.integrations.models import HttpExchange, Integration
 from readthedocs.oauth.services import registry

readthedocs/rtd_tests/tests/test_core_tags.py

@@ -1,4 +1,5 @@
 from unittest import mock
+
 import pytest
 from django.conf import settings
 from django.test import TestCase
@@ -30,10 +31,9 @@ def setUp(self):
         self.pip_latest_document_url = url_base.format(version='/en/latest/document')
         self.pip_latest_document_page_url = url_base.format(version='/en/latest/document.html')
 
-        with mock.patch('readthedocs.projects.models.broadcast'):
-            self.client.login(username='eric', password='test')
-            self.pip = Project.objects.get(slug='pip')
-            self.pip_fr = Project.objects.create(name='PIP-FR', slug='pip-fr', language='fr', main_language_project=self.pip)
+        self.client.login(username='eric', password='test')
+        self.pip = Project.objects.get(slug='pip')
+        self.pip_fr = Project.objects.create(name='PIP-FR', slug='pip-fr', language='fr', main_language_project=self.pip)
 
     def test_project_only(self):
         proj = Project.objects.get(slug='pip')

readthedocs/rtd_tests/tests/test_core_utils.py

@@ -1,22 +1,21 @@
 """Test core util functions."""
 
-import os
+from unittest import mock
 
 import pytest
-from unittest import mock
-from django.http import Http404
 from django.test import TestCase
 from django_dynamic_fixture import get
-from unittest.mock import call
 
-from readthedocs.builds.constants import LATEST, BUILD_STATE_BUILDING
-from readthedocs.builds.models import Version, Build
-from readthedocs.core.utils import slugify, trigger_build, prepare_build
-from readthedocs.core.utils.general import wipe_version_via_slugs
+from readthedocs.builds.constants import BUILD_STATE_BUILDING, LATEST
+from readthedocs.builds.models import Build, Version
+from readthedocs.core.utils import slugify, trigger_build
 from readthedocs.doc_builder.exceptions import BuildMaxConcurrencyError
-from readthedocs.projects.constants import CELERY_LOW, CELERY_MEDIUM, CELERY_HIGH
-from readthedocs.projects.models import Project, Feature
-from readthedocs.projects.tasks import remove_dirs
+from readthedocs.projects.constants import (
+    CELERY_HIGH,
+    CELERY_LOW,
+    CELERY_MEDIUM,
+)
+from readthedocs.projects.models import Feature, Project
 
 
 class CoreUtilTests(TestCase):
@@ -299,58 +298,3 @@ def test_slugify(self):
             slugify('A title_-_with separated parts', dns_safe=False),
             'a-title_-_with-separated-parts',
         )
-
-    @mock.patch('readthedocs.core.utils.general.broadcast')
-    def test_wipe_version_via_slug(self, mock_broadcast):
-        wipe_version_via_slugs(
-            version_slug=self.version.slug,
-            project_slug=self.version.project.slug
-        )
-        expected_del_dirs = [
-            os.path.join(self.version.project.doc_path, 'checkouts', self.version.slug),
-            os.path.join(self.version.project.doc_path, 'envs', self.version.slug),
-            os.path.join(self.version.project.doc_path, 'conda', self.version.slug),
-        ]
-
-        mock_broadcast.assert_has_calls(
-            [
-                call(type='build', task=remove_dirs, args=[(expected_del_dirs[0],)]),
-                call(type='build', task=remove_dirs, args=[(expected_del_dirs[1],)]),
-                call(type='build', task=remove_dirs, args=[(expected_del_dirs[2],)]),
-            ],
-            any_order=False
-        )
-
-    @mock.patch('readthedocs.core.utils.general.broadcast')
-    def test_wipe_version_via_slug_wrong_param(self, mock_broadcast):
-        self.assertFalse(Version.objects.filter(slug='wrong-slug').exists())
-        with self.assertRaises(Http404):
-            wipe_version_via_slugs(
-                version_slug='wrong-slug',
-                project_slug=self.version.project.slug
-            )
-        mock_broadcast.assert_not_called()
-
-    @mock.patch('readthedocs.core.utils.general.broadcast')
-    def test_wipe_version_via_slugs_same_version_slug_with_diff_proj(self, mock_broadcast):
-        project_2 = get(Project)
-        version_2 = get(Version, project=project_2, slug=self.version.slug)
-        wipe_version_via_slugs(
-            version_slug=version_2.slug,
-            project_slug=project_2.slug,
-        )
-
-        expected_del_dirs = [
-            os.path.join(version_2.project.doc_path, 'checkouts', version_2.slug),
-            os.path.join(version_2.project.doc_path, 'envs', version_2.slug),
-            os.path.join(version_2.project.doc_path, 'conda', version_2.slug),
-        ]
-
-        mock_broadcast.assert_has_calls(
-            [
-                call(type='build', task=remove_dirs, args=[(expected_del_dirs[0],)]),
-                call(type='build', task=remove_dirs, args=[(expected_del_dirs[1],)]),
-                call(type='build', task=remove_dirs, args=[(expected_del_dirs[2],)]),
-            ],
-            any_order=False
-        )