optimizing indexing

Author: safwanrahman

readthedocs/projects/signals.py

@@ -14,3 +14,7 @@
 project_import = django.dispatch.Signal(providing_args=["project"])
 
 files_changed = django.dispatch.Signal(providing_args=["project", "files"])
+
+bulk_post_create = django.dispatch.Signal(providing_args=["instance_list"])
+
+bulk_post_delete = django.dispatch.Signal(providing_args=["instance_list"])

readthedocs/projects/tasks.py

@@ -32,7 +32,8 @@
 from .constants import LOG_TEMPLATE
 from .exceptions import RepositoryError
 from .models import ImportedFile, Project, Domain, Feature, HTMLFile
-from .signals import before_vcs, after_vcs, before_build, after_build, files_changed
+from .signals import before_vcs, after_vcs, before_build, after_build, files_changed, \
+    bulk_post_create, bulk_post_delete
 from readthedocs.builds.constants import (
     BUILD_STATE_BUILDING, BUILD_STATE_CLONING, BUILD_STATE_FINISHED,
     BUILD_STATE_INSTALLING, LATEST, LATEST_VERBOSE_NAME, STABLE_VERBOSE_NAME)
@@ -986,6 +987,7 @@ def _manage_imported_files(version, path, commit):
     :param commit: Commit that updated path
     """
     changed_files = set()
+    created_html_files = []
     for root, __, filenames in os.walk(path):
         for filename in filenames:
             if fnmatch.fnmatch(filename, '*.html'):
@@ -1015,15 +1017,27 @@ def _manage_imported_files(version, path, commit):
                 obj.commit = commit
             obj.save()
 
-    # Delete the HTMLFile first from previous versions
-    HTMLFile.objects.filter(project=version.project,
-                            version=version
-                            ).exclude(commit=commit).delete()
+            if isinstance(obj, HTMLFile):
+                # the `obj` is HTMLFile, so add it to the list
+                created_html_files.append(obj)
+
+    # Send bulk_post_create signal for bulk indexing to Elasticsearch
+    bulk_post_create.send(sender=HTMLFile, instance_list=created_html_files)
+
+    # Delete the HTMLFile first from previous commit and
+    # send bulk_post_delete signal for bulk removing from Elasticsearch
+    delete_queryset = (HTMLFile.objects.filter(project=version.project, version=version)
+                                       .exclude(commit=commit))
+    # Keep the objects into memory to send it to signal
+    instance_list = list(delete_queryset)
+    # Safely delete from database
+    delete_queryset.delete()
+    # Always pass the list of instance, not queryset.
+    bulk_post_delete.send(sender=HTMLFile, instance_list=instance_list)
 
     # Delete ImportedFiles from previous versions
-    ImportedFile.objects.filter(project=version.project,
-                                version=version
-                                ).exclude(commit=commit).delete()
+    (ImportedFile.objects.filter(project=version.project, version=version)
+                         .exclude(commit=commit).delete())
     changed_files = [
         resolve_path(
             version.project, filename=file, version_slug=version.slug,

readthedocs/search/__init__.py

@@ -0,0 +1 @@
+default_app_config = 'readthedocs.search.apps.SearchConfig'

readthedocs/search/apps.py

@@ -0,0 +1,10 @@
+"""Project app config"""
+
+from django.apps import AppConfig
+
+
+class SearchConfig(AppConfig):
+    name = 'readthedocs.search'
+
+    def ready(self):
+        from .signals import index_html_file, remove_html_file

readthedocs/search/documents.py

@@ -52,6 +52,7 @@ class PageDocument(RTDDocTypeMixin, DocType):
     class Meta(object):
         model = HTMLFile
         fields = ('commit',)
+        ignore_signals = True
 
     project = fields.KeywordField(attr='project.slug')
     version = fields.KeywordField(attr='version.slug')

readthedocs/search/signals.py

@@ -1,7 +1,32 @@
 """We define custom Django signals to trigger before executing searches."""
 from __future__ import absolute_import
 import django.dispatch
+from django.dispatch import receiver
+from django_elasticsearch_dsl.registries import registry
+
+from readthedocs.projects.models import HTMLFile
+from readthedocs.projects.signals import bulk_post_create, bulk_post_delete
+from readthedocs.search.documents import PageDocument
+from readthedocs.search.tasks import index_objects_to_es_task
 
 before_project_search = django.dispatch.Signal(providing_args=["body"])
 before_file_search = django.dispatch.Signal(providing_args=["body"])
 before_section_search = django.dispatch.Signal(providing_args=["body"])
+
+
+@receiver(bulk_post_create, sender=HTMLFile)
+def index_html_file(instance_list, **_):
+    kwargs = {
+        'app_label': HTMLFile._meta.app_label,
+        'model_name': HTMLFile.__name__,
+        'document_class': str(PageDocument),
+        'index_name': None,  # No neeed to change the index name
+        'objects_id': [obj.id for obj in instance_list],
+    }
+
+    index_objects_to_es_task.delay(**kwargs)
+
+
+@receiver(bulk_post_delete, sender=HTMLFile)
+def remove_html_file(instance_list, **_):
+    registry.delete(instance_list)