Merge pull request #5290 from rtfd/domain-search

Add search for DomainData objects

Author: ericholscher

readthedocs/projects/admin.py

@@ -307,6 +307,7 @@ class ImportedFileAdmin(admin.ModelAdmin):
 
     raw_id_fields = ('project', 'version')
     list_display = ('path', 'name', 'version')
+    search_fields = ('project', 'path')
 
 
 class DomainAdmin(admin.ModelAdmin):

readthedocs/projects/models.py

@@ -13,6 +13,7 @@
 from django.db.models import Prefetch
 from django.urls import NoReverseMatch, reverse
 from django.utils.translation import ugettext_lazy as _
+from django.utils.functional import cached_property
 from django_extensions.db.models import TimeStampedModel
 from guardian.shortcuts import assign
 from six.moves import shlex_quote
@@ -1215,7 +1216,7 @@ def get_processed_json(self):
             'sections': [],
         }
 
-    @property
+    @cached_property
     def processed_json(self):
         return self.get_processed_json()
 

readthedocs/projects/signals.py

@@ -15,9 +15,9 @@
 
 files_changed = django.dispatch.Signal(providing_args=['project', 'files'])
 
-bulk_post_create = django.dispatch.Signal(providing_args=['instance_list'])
+bulk_post_create = django.dispatch.Signal(providing_args=['instance_list', 'commit'])
 
-bulk_post_delete = django.dispatch.Signal(providing_args=['instance_list'])
+bulk_post_delete = django.dispatch.Signal(providing_args=['instance_list', 'commit'])
 
 # Used to force verify a domain (eg. for SSL cert issuance)
 domain_verify = django.dispatch.Signal(providing_args=['domain'])

readthedocs/projects/tasks.py

@@ -1177,13 +1177,12 @@ def fileify(version_pk, commit):
     project = version.project
 
     if not commit:
-        log.info(
+        log.warning(
             LOG_TEMPLATE.format(
                 project=project.slug,
                 version=version.slug,
                 msg=(
-                    'Imported File not being built because no commit '
-                    'information'
+                    'Search index not being built because no commit information'
                 ),
             ),
         )
@@ -1198,16 +1197,15 @@ def fileify(version_pk, commit):
                 msg='Creating ImportedFiles',
             ),
         )
-        _manage_imported_files(version, path, commit)
-        _update_intersphinx_data(version, path, commit)
-    else:
-        log.info(
-            LOG_TEMPLATE.format(
-                project=project.slug,
-                version=version.slug,
-                msg='No ImportedFile files',
-            ),
-        )
+        try:
+            _manage_imported_files(version, path, commit)
+        except Exception:
+            log.exception('Failed during ImportedFile creation')
+
+        try:
+            _update_intersphinx_data(version, path, commit)
+        except Exception:
+            log.exception('Failed during SphinxDomain creation')
 
 
 def _update_intersphinx_data(version, path, commit):
@@ -1223,6 +1221,20 @@ def _update_intersphinx_data(version, path, commit):
         log.debug('No objects.inv, skipping intersphinx indexing.')
         return
 
+    full_json_path = version.project.get_production_media_path(
+        type_='json', version_slug=version.slug, include_file=False
+    )
+    type_file = os.path.join(full_json_path, 'readthedocs-sphinx-domain-names.json')
+    types = {}
+    titles = {}
+    if os.path.exists(type_file):
+        try:
+            data = json.load(open(type_file))
+            types = data['types']
+            titles = data['titles']
+        except Exception:
+            log.exception('Exception parsing readthedocs-sphinx-domain-names.json')
+
     # These classes are copied from Sphinx
     # https://git.io/fhFbI
     class MockConfig:
@@ -1236,6 +1248,8 @@ class MockApp:
         def warn(self, msg):
             log.warning('Sphinx MockApp: %s', msg)
 
+    created_sphinx_domains = []
+
     invdata = intersphinx.fetch_inventory(MockApp(), '', object_file)
     for key, value in sorted(invdata.items() or {}):
         domain, _type = key.split(':')
@@ -1252,22 +1266,41 @@ def warn(self, msg):
             else:
                 doc_name, anchor = url, ''
             display_name = einfo[3]
-            obj, _ = SphinxDomain.objects.get_or_create(
+            obj, created = SphinxDomain.objects.get_or_create(
                 project=version.project,
                 version=version,
                 domain=domain,
                 name=name,
                 display_name=display_name,
                 type=_type,
+                type_display=types.get(f'{domain}:{_type}', ''),
                 doc_name=doc_name,
+                doc_display=titles.get(doc_name, ''),
                 anchor=anchor,
             )
             if obj.commit != commit:
                 obj.commit = commit
                 obj.save()
-    SphinxDomain.objects.filter(project=version.project,
-                                version=version
-                                ).exclude(commit=commit).delete()
+            if created:
+                created_sphinx_domains.append(obj)
+
+    # Send bulk_post_create signal for bulk indexing to Elasticsearch
+    bulk_post_create.send(sender=SphinxDomain, instance_list=created_sphinx_domains, commit=commit)
+
+    # Delete the SphinxDomain first from previous commit and
+    # send bulk_post_delete signal for bulk removing from Elasticsearch
+    delete_queryset = (
+        SphinxDomain.objects.filter(project=version.project,
+                                    version=version
+                                    ).exclude(commit=commit)
+    )
+    # Keep the objects into memory to send it to signal
+    instance_list = list(delete_queryset)
+    # Always pass the list of instance, not queryset.
+    bulk_post_delete.send(sender=SphinxDomain, instance_list=instance_list, commit=commit)
+
+    # Delete from previous versions
+    delete_queryset.delete()
 
 
 def _manage_imported_files(version, path, commit):
@@ -1294,7 +1327,7 @@ def _manage_imported_files(version, path, commit):
             md5 = hashlib.md5(open(full_path, 'rb').read()).hexdigest()
             try:
                 # pylint: disable=unpacking-non-sequence
-                obj, __ = model_class.objects.get_or_create(
+                obj, created = model_class.objects.get_or_create(
                     project=version.project,
                     version=version,
                     path=dirpath,
@@ -1310,34 +1343,38 @@ def _manage_imported_files(version, path, commit):
                 obj.commit = commit
             obj.save()
 
-            if model_class == HTMLFile:
+            if created and model_class == HTMLFile:
                 # the `obj` is HTMLFile, so add it to the list
                 created_html_files.append(obj)
 
     # Send bulk_post_create signal for bulk indexing to Elasticsearch
-    bulk_post_create.send(sender=HTMLFile, instance_list=created_html_files)
+    bulk_post_create.send(sender=HTMLFile, instance_list=created_html_files,
+                          version=version, commit=commit)
 
     # Delete the HTMLFile first from previous commit and
     # send bulk_post_delete signal for bulk removing from Elasticsearch
     delete_queryset = (
         HTMLFile.objects.filter(project=version.project,
                                 version=version).exclude(commit=commit)
     )
+
     # Keep the objects into memory to send it to signal
     instance_list = list(delete_queryset)
+
     # Always pass the list of instance, not queryset.
     # These objects must exist though,
     # because the task will query the DB for the objects before deleting
-    bulk_post_delete.send(sender=HTMLFile, instance_list=instance_list)
-    # Safely delete from database
-    delete_queryset.delete()
+    bulk_post_delete.send(sender=HTMLFile, instance_list=instance_list,
+                          version=version, commit=commit)
 
     # Delete ImportedFiles from previous versions
-    (
-        ImportedFile.objects.filter(project=version.project,
-                                    version=version).exclude(commit=commit
-                                                             ).delete()
-    )
+    delete_queryset.delete()
+
+    # This is required to delete ImportedFile objects that aren't HTMLFile objects,
+    ImportedFile.objects.filter(
+        project=version.project, version=version
+    ).exclude(commit=commit).delete()
+
     changed_files = [
         resolve_path(
             version.project,

readthedocs/projects/urls/public.py

@@ -51,7 +51,7 @@
     ),
     url(
         r'^(?P<project_slug>{project_slug})/search/$'.format(**pattern_opts),
-        search_views.elastic_project_search,
+        search_views.elastic_search,
         name='elastic_project_search',
     ),
     url(

readthedocs/restapi/urls.py

@@ -35,7 +35,7 @@
 router.register(r'project', ProjectViewSet, basename='project')
 router.register(r'notification', NotificationViewSet, basename='emailhook')
 router.register(r'domain', DomainViewSet, basename='domain')
-router.register(r'sphinx_domains', SphinxDomainAPIView, basename='sphinxdomain')
+router.register(r'sphinx_domain', SphinxDomainAPIView, basename='sphinxdomain')
 router.register(
     r'remote/org',
     RemoteOrganizationViewSet,

readthedocs/search/api.py

@@ -6,14 +6,14 @@
 from rest_framework.exceptions import ValidationError
 from rest_framework.pagination import PageNumberPagination
 
-from readthedocs.search.documents import PageDocument
+from readthedocs.search.faceted_search import PageSearch
 from readthedocs.search.utils import get_project_list_or_404
 
 log = logging.getLogger(__name__)
 
 
 class SearchPagination(PageNumberPagination):
-    page_size = 25
+    page_size = 50
     page_size_query_param = 'page_size'
     max_page_size = 100
 
@@ -62,15 +62,15 @@ def get_queryset(self):
         # Validate all the required params are there
         self.validate_query_params()
         query = self.request.query_params.get('q', '')
-        kwargs = {'filter_by_user': False}
-        kwargs['projects_list'] = [p.slug for p in self.get_all_projects()]
-        kwargs['versions_list'] = self.request.query_params.get('version')
-        if not kwargs['projects_list']:
+        kwargs = {'filter_by_user': False, 'filters': {}}
+        kwargs['filters']['project'] = [p.slug for p in self.get_all_projects()]
+        kwargs['filters']['version'] = self.request.query_params.get('version')
+        if not kwargs['filters']['project']:
             raise ValidationError("Unable to find a project to search")
-        if not kwargs['versions_list']:
+        if not kwargs['filters']['version']:
             raise ValidationError("Unable to find a version to search")
         user = self.request.user
-        queryset = PageDocument.faceted_search(
+        queryset = PageSearch(
             query=query, user=user, **kwargs
         )
         return queryset

readthedocs/search/documents.py

@@ -1,10 +1,10 @@
-# -*- coding: utf-8 -*-
 import logging
 
 from django.conf import settings
 from django_elasticsearch_dsl import DocType, Index, fields
 
 from readthedocs.projects.models import HTMLFile, Project
+from readthedocs.sphinx_domains.models import SphinxDomain
 
 
 project_conf = settings.ES_INDEXES['project']
@@ -15,9 +15,53 @@
 page_index = Index(page_conf['name'])
 page_index.settings(**page_conf['settings'])
 
+domain_conf = settings.ES_INDEXES['domain']
+domain_index = Index(domain_conf['name'])
+domain_index.settings(**domain_conf['settings'])
+
 log = logging.getLogger(__name__)
 
 
+@domain_index.doc_type
+class SphinxDomainDocument(DocType):
+    project = fields.KeywordField(attr='project.slug')
+    version = fields.KeywordField(attr='version.slug')
+    role_name = fields.KeywordField(attr='role_name')
+
+    # For linking to the URL
+    doc_name = fields.KeywordField(attr='doc_name')
+    anchor = fields.KeywordField(attr='anchor')
+
+    # For showing in the search result
+    type_display = fields.TextField(attr='type_display')
+    doc_display = fields.TextField(attr='doc_display')
+
+    # Simple analyzer breaks on `.`,
+    # otherwise search results are too strict for this use case
+    name = fields.TextField(attr='name', analyzer='simple')
+    display_name = fields.TextField(attr='display_name', analyzer='simple')
+
+    modified_model_field = 'modified'
+
+    class Meta(object):
+        model = SphinxDomain
+        fields = ('commit',)
+        ignore_signals = True
+
+    def get_queryset(self):
+        """Overwrite default queryset to filter certain files to index."""
+        queryset = super().get_queryset()
+
+        excluded_types = [
+            {'domain': 'std', 'type': 'doc'},
+            {'domain': 'std', 'type': 'label'},
+        ]
+
+        for exclude in excluded_types:
+            queryset = queryset.exclude(**exclude)
+        return queryset
+
+
 @project_index.doc_type
 class ProjectDocument(DocType):
 
@@ -31,6 +75,8 @@ class ProjectDocument(DocType):
     )
     language = fields.KeywordField()
 
+    modified_model_field = 'modified_date'
+
     class Meta(object):
         model = Project
         fields = ('name', 'slug', 'description')
@@ -63,6 +109,8 @@ class PageDocument(DocType):
     headers = fields.TextField(attr='processed_json.headers')
     content = fields.TextField(attr='processed_json.content')
 
+    modified_model_field = 'modified_date'
+
     class Meta(object):
         model = HTMLFile
         fields = ('commit',)