diff --git a/readthedocs/builds/admin.py b/readthedocs/builds/admin.py index af83ba82f12..1888dc70fc9 100644 --- a/readthedocs/builds/admin.py +++ b/readthedocs/builds/admin.py @@ -7,6 +7,8 @@ from readthedocs.builds.models import Build, BuildCommandResult, Version from readthedocs.core.utils import trigger_build +from readthedocs.projects.models import HTMLFile +from readthedocs.search.utils import _indexing_helper from readthedocs.core.utils.general import wipe_version_via_slugs @@ -58,7 +60,7 @@ class VersionAdmin(GuardedModelAdmin): list_filter = ('type', 'privacy_level', 'active', 'built') search_fields = ('slug', 'project__slug') raw_id_fields = ('project',) - actions = ['wipe_selected_versions', 'build_version'] + actions = ['build_version', 'reindex_version', 'wipe_version', 'wipe_selected_versions'] def wipe_selected_versions(self, request, queryset): """Wipes the selected versions.""" @@ -92,6 +94,46 @@ def build_version(self, request, queryset): build_version.short_description = 'Build version' + def reindex_version(self, request, queryset): + """Reindexes all selected versions to ES.""" + html_objs_qs = [] + for version in queryset.iterator(): + html_objs = HTMLFile.objects.filter(project=version.project, version=version) + + if html_objs.exists(): + html_objs_qs.append(html_objs) + + if html_objs_qs: + _indexing_helper(html_objs_qs, wipe=False) + + self.message_user( + request, + 'Task initiated successfully.', + messages.SUCCESS + ) + + reindex_version.short_description = 'Reindex version to ES' + + def wipe_version(self, request, queryset): + """Wipe selected versions from ES.""" + html_objs_qs = [] + for version in queryset.iterator(): + html_objs = HTMLFile.objects.filter(project=version.project, version=version) + + if html_objs.exists(): + html_objs_qs.append(html_objs) + + if html_objs_qs: + _indexing_helper(html_objs_qs, wipe=True) + + self.message_user( + request, + 'Task initiated successfully', + messages.SUCCESS, + ) + + wipe_version.short_description = 'Wipe version from ES' + admin.site.register(Build, BuildAdmin) admin.site.register(Version, VersionAdmin) diff --git a/readthedocs/projects/admin.py b/readthedocs/projects/admin.py index db29088aa9f..7f6da9d88fe 100644 --- a/readthedocs/projects/admin.py +++ b/readthedocs/projects/admin.py @@ -12,6 +12,7 @@ from readthedocs.core.utils import broadcast, trigger_build from readthedocs.notifications.views import SendNotificationView from readthedocs.redirects.models import Redirect +from readthedocs.search.utils import _indexing_helper from .forms import FeatureForm from .models import ( @@ -145,7 +146,13 @@ class ProjectAdmin(GuardedModelAdmin): ] readonly_fields = ('feature_flags',) raw_id_fields = ('users', 'main_language_project') - actions = ['send_owner_email', 'ban_owner', 'build_default_version'] + actions = [ + 'send_owner_email', + 'ban_owner', + 'build_default_version', + 'reindex_active_versions', + 'wipe_all_versions', + ] def feature_flags(self, obj): return ', '.join([str(f.get_feature_display()) for f in obj.features]) @@ -222,6 +229,68 @@ def build_default_version(self, request, queryset): build_default_version.short_description = 'Build default version' + def reindex_active_versions(self, request, queryset): + """Reindex all active versions of the selected projects to ES.""" + qs_iterator = queryset.iterator() + for project in qs_iterator: + version_qs = Version.objects.filter(project=project) + active_versions = version_qs.filter(active=True) + + if not active_versions.exists(): + self.message_user( + request, + 'No active versions of project {}'.format(project), + messages.ERROR + ) + else: + html_objs_qs = [] + for version in active_versions.iterator(): + html_objs = HTMLFile.objects.filter(project=project, version=version) + + if html_objs.exists(): + html_objs_qs.append(html_objs) + + if html_objs_qs: + _indexing_helper(html_objs_qs, wipe=False) + + self.message_user( + request, + 'Task initiated successfully for {}'.format(project), + messages.SUCCESS + ) + + reindex_active_versions.short_description = 'Reindex active versions to ES' + + def wipe_all_versions(self, request, queryset): + """Wipe indexes of all versions of selected projects.""" + qs_iterator = queryset.iterator() + for project in qs_iterator: + version_qs = Version.objects.filter(project=project) + if not version_qs.exists(): + self.message_user( + request, + 'No active versions of project {}.'.format(project), + messages.ERROR + ) + else: + html_objs_qs = [] + for version in version_qs.iterator(): + html_objs = HTMLFile.objects.filter(project=project, version=version) + + if html_objs.exists(): + html_objs_qs.append(html_objs) + + if html_objs_qs: + _indexing_helper(html_objs_qs, wipe=True) + + self.message_user( + request, + 'Task initiated successfully for {}.'.format(project), + messages.SUCCESS + ) + + wipe_all_versions.short_description = 'Wipe all versions from ES' + def get_actions(self, request): actions = super().get_actions(request) actions['delete_selected'] = ( diff --git a/readthedocs/search/utils.py b/readthedocs/search/utils.py index 34dc2c092a7..5f04f6736b7 100644 --- a/readthedocs/search/utils.py +++ b/readthedocs/search/utils.py @@ -8,7 +8,8 @@ from django_elasticsearch_dsl.registries import registry from readthedocs.builds.models import Version -from readthedocs.projects.models import Project +from readthedocs.projects.models import Project, HTMLFile +from readthedocs.search.documents import PageDocument log = logging.getLogger(__name__) @@ -67,3 +68,34 @@ def _get_document(model, document_class): for document in documents: if str(document) == document_class: return document + + +def _indexing_helper(html_objs_qs, wipe=False): + """ + Helper function for reindexing and wiping indexes of projects and versions. + + If ``wipe`` is set to False, html_objs are deleted from the ES index, + else, html_objs are indexed. + """ + from readthedocs.search.tasks import index_objects_to_es, delete_objects_in_es + + if html_objs_qs: + obj_ids = [] + for html_objs in html_objs_qs: + obj_ids.extend([obj.id for obj in html_objs]) + + # removing redundant ids if exists. + obj_ids = list(set(obj_ids)) + + if obj_ids: + kwargs = { + 'app_label': HTMLFile._meta.app_label, + 'model_name': HTMLFile.__name__, + 'document_class': str(PageDocument), + 'objects_id': obj_ids, + } + + if not wipe: + index_objects_to_es.delay(**kwargs) + else: + delete_objects_in_es.delay(**kwargs)