Add admin methods for reindexing versions from project and version admin. (#5343)

* Add admin methods

* add else

* add another else

* Change short_description

* add improvements

Author: dojutsu-user

readthedocs/builds/admin.py

@@ -7,6 +7,8 @@
 
 from readthedocs.builds.models import Build, BuildCommandResult, Version
 from readthedocs.core.utils import trigger_build
+from readthedocs.projects.models import HTMLFile
+from readthedocs.search.utils import _indexing_helper
 from readthedocs.core.utils.general import wipe_version_via_slugs
 
 
@@ -58,7 +60,7 @@ class VersionAdmin(GuardedModelAdmin):
     list_filter = ('type', 'privacy_level', 'active', 'built')
     search_fields = ('slug', 'project__slug')
     raw_id_fields = ('project',)
-    actions = ['wipe_selected_versions', 'build_version']
+    actions = ['build_version', 'reindex_version', 'wipe_version', 'wipe_selected_versions']
 
     def wipe_selected_versions(self, request, queryset):
         """Wipes the selected versions."""
@@ -92,6 +94,46 @@ def build_version(self, request, queryset):
 
     build_version.short_description = 'Build version'
 
+    def reindex_version(self, request, queryset):
+        """Reindexes all selected versions to ES."""
+        html_objs_qs = []
+        for version in queryset.iterator():
+            html_objs = HTMLFile.objects.filter(project=version.project, version=version)
+
+            if html_objs.exists():
+                html_objs_qs.append(html_objs)
+
+        if html_objs_qs:
+            _indexing_helper(html_objs_qs, wipe=False)
+
+        self.message_user(
+            request,
+            'Task initiated successfully.',
+            messages.SUCCESS
+        )
+
+    reindex_version.short_description = 'Reindex version to ES'
+
+    def wipe_version(self, request, queryset):
+        """Wipe selected versions from ES."""
+        html_objs_qs = []
+        for version in queryset.iterator():
+            html_objs = HTMLFile.objects.filter(project=version.project, version=version)
+
+            if html_objs.exists():
+                html_objs_qs.append(html_objs)
+
+        if html_objs_qs:
+            _indexing_helper(html_objs_qs, wipe=True)
+
+        self.message_user(
+            request,
+            'Task initiated successfully',
+            messages.SUCCESS,
+        )
+
+    wipe_version.short_description = 'Wipe version from ES'
+
 
 admin.site.register(Build, BuildAdmin)
 admin.site.register(Version, VersionAdmin)

readthedocs/projects/admin.py

@@ -12,6 +12,7 @@
 from readthedocs.core.utils import broadcast, trigger_build
 from readthedocs.notifications.views import SendNotificationView
 from readthedocs.redirects.models import Redirect
+from readthedocs.search.utils import _indexing_helper
 
 from .forms import FeatureForm
 from .models import (
@@ -145,7 +146,13 @@ class ProjectAdmin(GuardedModelAdmin):
     ]
     readonly_fields = ('feature_flags',)
     raw_id_fields = ('users', 'main_language_project')
-    actions = ['send_owner_email', 'ban_owner', 'build_default_version']
+    actions = [
+        'send_owner_email',
+        'ban_owner',
+        'build_default_version',
+        'reindex_active_versions',
+        'wipe_all_versions',
+    ]
 
     def feature_flags(self, obj):
         return ', '.join([str(f.get_feature_display()) for f in obj.features])
@@ -222,6 +229,68 @@ def build_default_version(self, request, queryset):
 
     build_default_version.short_description = 'Build default version'
 
+    def reindex_active_versions(self, request, queryset):
+        """Reindex all active versions of the selected projects to ES."""
+        qs_iterator = queryset.iterator()
+        for project in qs_iterator:
+            version_qs = Version.objects.filter(project=project)
+            active_versions = version_qs.filter(active=True)
+
+            if not active_versions.exists():
+                self.message_user(
+                    request,
+                    'No active versions of project {}'.format(project),
+                    messages.ERROR
+                )
+            else:
+                html_objs_qs = []
+                for version in active_versions.iterator():
+                    html_objs = HTMLFile.objects.filter(project=project, version=version)
+
+                    if html_objs.exists():
+                        html_objs_qs.append(html_objs)
+
+                if html_objs_qs:
+                    _indexing_helper(html_objs_qs, wipe=False)
+
+                self.message_user(
+                    request,
+                    'Task initiated successfully for {}'.format(project),
+                    messages.SUCCESS
+                )
+
+    reindex_active_versions.short_description = 'Reindex active versions to ES'
+
+    def wipe_all_versions(self, request, queryset):
+        """Wipe indexes of all versions of selected projects."""
+        qs_iterator = queryset.iterator()
+        for project in qs_iterator:
+            version_qs = Version.objects.filter(project=project)
+            if not version_qs.exists():
+                self.message_user(
+                    request,
+                    'No active versions of project {}.'.format(project),
+                    messages.ERROR
+                )
+            else:
+                html_objs_qs = []
+                for version in version_qs.iterator():
+                    html_objs = HTMLFile.objects.filter(project=project, version=version)
+
+                    if html_objs.exists():
+                        html_objs_qs.append(html_objs)
+
+                if html_objs_qs:
+                    _indexing_helper(html_objs_qs, wipe=True)
+
+                self.message_user(
+                    request,
+                    'Task initiated successfully for {}.'.format(project),
+                    messages.SUCCESS
+                )
+
+    wipe_all_versions.short_description = 'Wipe all versions from ES'
+
     def get_actions(self, request):
         actions = super().get_actions(request)
         actions['delete_selected'] = (

readthedocs/search/utils.py

@@ -8,7 +8,8 @@
 from django_elasticsearch_dsl.registries import registry
 
 from readthedocs.builds.models import Version
-from readthedocs.projects.models import Project
+from readthedocs.projects.models import Project, HTMLFile
+from readthedocs.search.documents import PageDocument
 
 
 log = logging.getLogger(__name__)
@@ -67,3 +68,34 @@ def _get_document(model, document_class):
     for document in documents:
         if str(document) == document_class:
             return document
+
+
+def _indexing_helper(html_objs_qs, wipe=False):
+    """
+    Helper function for reindexing and wiping indexes of projects and versions.
+
+    If ``wipe`` is set to False, html_objs are deleted from the ES index,
+    else, html_objs are indexed.
+    """
+    from readthedocs.search.tasks import index_objects_to_es, delete_objects_in_es
+
+    if html_objs_qs:
+        obj_ids = []
+        for html_objs in html_objs_qs:
+            obj_ids.extend([obj.id for obj in html_objs])
+
+        # removing redundant ids if exists.
+        obj_ids = list(set(obj_ids))
+
+        if obj_ids:
+            kwargs = {
+                'app_label': HTMLFile._meta.app_label,
+                'model_name': HTMLFile.__name__,
+                'document_class': str(PageDocument),
+                'objects_id': obj_ids,
+            }
+
+            if not wipe:
+                index_objects_to_es.delay(**kwargs)
+            else:
+                delete_objects_in_es.delay(**kwargs)