Skip to content

Add admin methods for reindexing versions from project and version admin. #5343

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Feb 27, 2019
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 31 additions & 1 deletion readthedocs/builds/admin.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@

from readthedocs.builds.models import Build, BuildCommandResult, Version
from readthedocs.core.utils import trigger_build
from readthedocs.projects.models import HTMLFile
from readthedocs.search.utils import _indexing_helper
from readthedocs.core.utils.general import wipe_version_via_slugs


Expand Down Expand Up @@ -58,7 +60,7 @@ class VersionAdmin(GuardedModelAdmin):
list_filter = ('type', 'privacy_level', 'active', 'built')
search_fields = ('slug', 'project__slug')
raw_id_fields = ('project',)
actions = ['wipe_selected_versions', 'build_version']
actions = ['build_version', 'reindex_version', 'wipe_version', 'wipe_selected_versions']

def wipe_selected_versions(self, request, queryset):
"""Wipes the selected versions."""
Expand Down Expand Up @@ -92,6 +94,34 @@ def build_version(self, request, queryset):

build_version.short_description = 'Build version'

def reindex_version(self, request, queryset):
"""Reindexes all selected versions to ES."""
for version in queryset.iterator():
html_objs = HTMLFile.objects.filter(project=version.project, version=version)
_indexing_helper(html_objs, wipe=False)

self.message_user(
request,
'Task initiated successfully',
messages.SUCCESS
)

reindex_version.short_description = 'Reindex version to ES'

def wipe_version(self, request, queryset):
"""Wipe selected versions from ES."""
for version in queryset.iterator():
html_objs = HTMLFile.objects.filter(project=version.project, version=version)
_indexing_helper(html_objs, wipe=True)

self.message_user(
request,
'Task initiated successfully',
messages.SUCCESS,
)

wipe_version.short_description = 'Wipe version from ES'


admin.site.register(Build, BuildAdmin)
admin.site.register(Version, VersionAdmin)
59 changes: 58 additions & 1 deletion readthedocs/projects/admin.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
from readthedocs.core.utils import broadcast, trigger_build
from readthedocs.notifications.views import SendNotificationView
from readthedocs.redirects.models import Redirect
from readthedocs.search.utils import _indexing_helper

from .forms import FeatureForm
from .models import (
Expand Down Expand Up @@ -145,7 +146,13 @@ class ProjectAdmin(GuardedModelAdmin):
]
readonly_fields = ('feature_flags',)
raw_id_fields = ('users', 'main_language_project')
actions = ['send_owner_email', 'ban_owner', 'build_default_version']
actions = [
'send_owner_email',
'ban_owner',
'build_default_version',
'reindex_active_versions',
'wipe_all_versions',
]

def feature_flags(self, obj):
return ', '.join([str(f.get_feature_display()) for f in obj.features])
Expand Down Expand Up @@ -222,6 +229,56 @@ def build_default_version(self, request, queryset):

build_default_version.short_description = 'Build default version'

def reindex_active_versions(self, request, queryset):
"""Reindex all active versions of the selected projects to ES."""
qs_iterator = queryset.iterator()
for project in qs_iterator:
version_qs = Version.objects.filter(project=project)
active_versions = version_qs.filter(active=True)

if not active_versions.exists():
self.message_user(
request,
'No active versions of project {}'.format(project),
messages.ERROR
)
else:
for version in active_versions.iterator():
html_objs = HTMLFile.objects.filter(project=project, version=version)
_indexing_helper(html_objs, wipe=False)
Copy link
Member

@ericholscher ericholscher Feb 26, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is reindexing a number of times, it should build a list of all the files and then pass it once.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thank you.
I have made and pushed the changes.


self.message_user(
request,
'Task initiated successfully.',
messages.SUCCESS
)

reindex_active_versions.short_description = 'Reindex active versions to ES'

def wipe_all_versions(self, request, queryset):
"""Wipe indexes of all versions of selected projects."""
qs_iterator = queryset.iterator()
for project in qs_iterator:
version_qs = Version.objects.filter(project=project)
if not version_qs.exists():
self.message_user(
request,
'No active versions of project {}'.format(project),
messages.ERROR
)
else:
for version in version_qs.iterator():
html_objs = HTMLFile.objects.filter(project=project, version=version)
_indexing_helper(html_objs, wipe=True)

self.message_user(
request,
'Task initiated successfully.',
messages.SUCCESS
)

wipe_all_versions.short_description = 'Wipe all versions from ES'

def get_actions(self, request):
actions = super().get_actions(request)
actions['delete_selected'] = (
Expand Down
25 changes: 24 additions & 1 deletion readthedocs/search/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@
from django_elasticsearch_dsl.registries import registry

from readthedocs.builds.models import Version
from readthedocs.projects.models import Project
from readthedocs.projects.models import Project, HTMLFile
from readthedocs.search.documents import PageDocument


log = logging.getLogger(__name__)
Expand Down Expand Up @@ -67,3 +68,25 @@ def _get_document(model, document_class):
for document in documents:
if str(document) == document_class:
return document


def _indexing_helper(html_objs, wipe=False):
"""
Helper function for reindexing and wiping indexes of projects and versions.

If ``wipe`` is set to False, html_objs are deleted from the ES index,
else, html_objs are indexed.
"""
from readthedocs.search.tasks import index_objects_to_es, delete_objects_in_es
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This should check if html_objs exists -- it might not.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I have added the checks to prevent this.


kwargs = {
'app_label': HTMLFile._meta.app_label,
'model_name': HTMLFile.__name__,
'document_class': str(PageDocument),
'objects_id': [obj.id for obj in html_objs],
}

if not wipe:
index_objects_to_es.delay(**kwargs)
else:
delete_objects_in_es.delay(**kwargs)