Make indexing work for SphinxDomain objects.

Author: ericholscher

readthedocs/projects/models.py

@@ -12,6 +12,7 @@
 from django.db import models
 from django.urls import NoReverseMatch, reverse
 from django.utils.translation import ugettext_lazy as _
+from django.utils.functional import cached_property
 from django_extensions.db.models import TimeStampedModel
 from guardian.shortcuts import assign
 from six.moves import shlex_quote
@@ -1183,7 +1184,7 @@ def get_processed_json(self):
             'sections': [],
         }
 
-    @property
+    @cached_property
     def processed_json(self):
         return self.get_processed_json()
 

readthedocs/projects/tasks.py

@@ -60,6 +60,7 @@
 )
 from readthedocs.doc_builder.loader import get_builder_class
 from readthedocs.doc_builder.python_environments import Conda, Virtualenv
+from readthedocs.search.documents import PageDocument, SphinxDomainDocument
 from readthedocs.sphinx_domains.models import SphinxDomain
 from readthedocs.projects.models import APIProject
 from readthedocs.restapi.client import api as api_v2
@@ -1213,13 +1214,12 @@ def fileify(version_pk, commit):
     project = version.project
 
     if not commit:
-        log.info(
+        log.warning(
             LOG_TEMPLATE.format(
                 project=project.slug,
                 version=version.slug,
                 msg=(
-                    'Imported File not being built because no commit '
-                    'information'
+                    'Search index not being built because no commit information'
                 ),
             ),
         )
@@ -1234,16 +1234,15 @@ def fileify(version_pk, commit):
                 msg='Creating ImportedFiles',
             ),
         )
-        _manage_imported_files(version, path, commit)
-        _update_intersphinx_data(version, path, commit)
-    else:
-        log.info(
-            LOG_TEMPLATE.format(
-                project=project.slug,
-                version=version.slug,
-                msg='No ImportedFile files',
-            ),
-        )
+        try:
+            _manage_imported_files(version, path, commit)
+        except Exception:
+            log.exception('Failed during ImportedFile creation')
+
+        try:
+            _update_intersphinx_data(version, path, commit)
+        except Exception:
+            log.exception('Failed during SphinxDomain creation')
 
 
 def _update_intersphinx_data(version, path, commit):
@@ -1286,6 +1285,8 @@ class MockApp:
         def warn(self, msg):
             log.warning('Sphinx MockApp: %s', msg)
 
+    created_sphinx_domains = []
+
     invdata = intersphinx.fetch_inventory(MockApp(), '', object_file)
     for key, value in sorted(invdata.items() or {}):
         domain, _type = key.split(':')
@@ -1302,7 +1303,7 @@ def warn(self, msg):
             else:
                 doc_name, anchor = url, ''
             display_name = einfo[3]
-            obj, _ = SphinxDomain.objects.get_or_create(
+            obj, created = SphinxDomain.objects.get_or_create(
                 project=version.project,
                 version=version,
                 domain=domain,
@@ -1317,9 +1318,25 @@ def warn(self, msg):
             if obj.commit != commit:
                 obj.commit = commit
                 obj.save()
-    SphinxDomain.objects.filter(project=version.project,
-                                version=version
-                                ).exclude(commit=commit).delete()
+            created_sphinx_domains.append(obj)
+
+    # Send bulk_post_create signal for bulk indexing to Elasticsearch
+    bulk_post_create.send(sender=SphinxDomainDocument, instance_list=created_sphinx_domains)
+
+    # Delete the HTMLFile first from previous commit and
+    # send bulk_post_delete signal for bulk removing from Elasticsearch
+    delete_queryset = (
+        SphinxDomain.objects.filter(project=version.project,
+                                    version=version
+                                    ).exclude(commit=commit)
+    )
+    # Keep the objects into memory to send it to signal
+    instance_list = list(delete_queryset)
+    # Always pass the list of instance, not queryset.
+    bulk_post_delete.send(sender=SphinxDomainDocument, instance_list=instance_list)
+
+    # Delete from previous versions
+    delete_queryset.delete()
 
 
 def _manage_imported_files(version, path, commit):
@@ -1367,7 +1384,7 @@ def _manage_imported_files(version, path, commit):
                 created_html_files.append(obj)
 
     # Send bulk_post_create signal for bulk indexing to Elasticsearch
-    bulk_post_create.send(sender=HTMLFile, instance_list=created_html_files)
+    bulk_post_create.send(sender=PageDocument, instance_list=created_html_files)
 
     # Delete the HTMLFile first from previous commit and
     # send bulk_post_delete signal for bulk removing from Elasticsearch
@@ -1377,10 +1394,8 @@ def _manage_imported_files(version, path, commit):
     )
     # Keep the objects into memory to send it to signal
     instance_list = list(delete_queryset)
-    # Safely delete from database
-    delete_queryset.delete()
     # Always pass the list of instance, not queryset.
-    bulk_post_delete.send(sender=HTMLFile, instance_list=instance_list)
+    bulk_post_delete.send(sender=PageDocument, instance_list=instance_list)
 
     # Delete ImportedFiles from previous versions
     delete_queryset.delete()

readthedocs/search/documents.py

@@ -27,16 +27,23 @@ class SphinxDomainDocument(DocType):
     project = fields.KeywordField(attr='project.slug')
     version = fields.KeywordField(attr='version.slug')
     role_name = fields.KeywordField(attr='role_name')
+
+    # For linking to the URL
+    doc_name = fields.KeywordField(attr='doc_name')
     anchor = fields.KeywordField(attr='anchor')
-    doc_display = fields.KeywordField(attr='doc_display')
-    type_display = fields.KeywordField(attr='type_display')
+
+    # For showing in the search result
+    type_display = fields.TextField(attr='type_display')
+    doc_display = fields.TextField(attr='doc_display')
+    # Simple analyzer breaks on `.`,
+    # otherwise search results are too strict for this use case
     name = fields.TextField(attr='name', analyzer='simple')
+    display_name = fields.TextField(attr='display_name', analyzer='simple')
 
     modified_model_field = 'modified'
 
     class Meta(object):
         model = SphinxDomain
-        fields = ('display_name', 'doc_name')
         ignore_signals = True
 
     def get_queryset(self):

readthedocs/search/faceted_search.py

@@ -104,7 +104,7 @@ class DomainSearchBase(RTDFacetedSearch):
     }
     doc_types = [SphinxDomainDocument]
     index = SphinxDomainDocument._doc_type.index
-    fields = ('display_name^5', 'name', 'project', 'type_display')
+    fields = ('display_name^5', 'name^3', 'project^3', 'type_display')
     operators = ['and']
 
 
@@ -152,7 +152,8 @@ class AllSearch(RTDFacetedSearch):
         'version': TermsFacet(field='version'),
         'language': TermsFacet(field='language'),
         'role_name': TermsFacet(field='role_name'),
-        'index': TermsFacet(field='_index'),
+        # Need to improve UX here for exposing to users
+        # 'index': TermsFacet(field='_index'),
     }
     doc_types = [SphinxDomainDocument, PageDocument, ProjectDocument]
     index = [SphinxDomainDocument._doc_type.index,

readthedocs/search/signals.py

@@ -5,19 +5,19 @@
 from django.dispatch import receiver
 from django_elasticsearch_dsl.apps import DEDConfig
 
-from readthedocs.projects.models import HTMLFile, Project
+from readthedocs.projects.models import Project
 from readthedocs.projects.signals import bulk_post_create, bulk_post_delete
 from readthedocs.search.tasks import delete_objects_in_es, index_objects_to_es
 
 
-@receiver(bulk_post_create, sender=HTMLFile)
-def index_html_file(instance_list, **_):
+@receiver(bulk_post_create)
+def index_indexed_file(sender, instance_list, **_):
     """Handle indexing from the build process."""
-    from readthedocs.search.documents import PageDocument
+    model = sender._doc_type.model
     kwargs = {
-        'app_label': HTMLFile._meta.app_label,
-        'model_name': HTMLFile.__name__,
-        'document_class': str(PageDocument),
+        'app_label': model._meta.app_label,
+        'model_name': model.__name__,
+        'document_class': str(sender),
         'objects_id': [obj.id for obj in instance_list],
     }
 
@@ -26,14 +26,14 @@ def index_html_file(instance_list, **_):
         index_objects_to_es(**kwargs)
 
 
-@receiver(bulk_post_delete, sender=HTMLFile)
-def remove_html_file(instance_list, **_):
+@receiver(bulk_post_delete)
+def remove_indexed_file(sender, instance_list, **_):
     """Remove deleted files from the build process."""
-    from readthedocs.search.documents import PageDocument
+    model = sender._doc_type.model
     kwargs = {
-        'app_label': HTMLFile._meta.app_label,
-        'model_name': HTMLFile.__name__,
-        'document_class': str(PageDocument),
+        'app_label': model._meta.app_label,
+        'model_name': model.__name__,
+        'document_class': str(sender),
         'objects_id': [obj.id for obj in instance_list],
     }
 

readthedocs/search/views.py

@@ -40,16 +40,13 @@ def elastic_search(request, project_slug=None):
     :param project_slug: Sent when the view is a project search
     """
 
-    request_type = request.GET.get('type', 'project')
-
     if project_slug:
         queryset = Project.objects.protected(request.user)
         project_obj = get_object_or_404(queryset, slug=project_slug)
-        request_type = 'all'
 
     user_input = UserInput(
         query=request.GET.get('q'),
-        type=request_type,
+        type=request.GET.get('type', 'file'),
         project=project_slug or request.GET.get('project'),
         version=request.GET.get('version', LATEST),
         taxonomy=request.GET.get('taxonomy'),

readthedocs/settings/base.py

@@ -328,7 +328,6 @@ def USE_PROMOS(self):  # noqa
 
     # CORS
     CORS_ORIGIN_REGEX_WHITELIST = (
-        '(.*)localhost(.*)',
         '^http://(.+)\.readthedocs\.io$',
         '^https://(.+)\.readthedocs\.io$'
     )

readthedocs/sphinx_domains/migrations/0003_additional_indexed_fields.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-03-01 14:25
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sphinx_domains', '0002_increase_max_length'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='sphinxdomain',
+            name='doc_display',
+            field=models.CharField(max_length=4092, null=True, verbose_name='Doc Display'),
+        ),
+        migrations.AddField(
+            model_name='sphinxdomain',
+            name='type_display',
+            field=models.CharField(max_length=4092, null=True, verbose_name='Type Display'),
+        ),
+    ]

readthedocs/sphinx_domains/models.py

@@ -53,6 +53,7 @@ class SphinxDomain(TimeStampedModel):
     type_display = models.CharField(
         _('Type Display'),
         max_length=4092,
+        null=True,
     )
     doc_name = models.CharField(
         _('Doc Name'),
@@ -61,6 +62,7 @@ class SphinxDomain(TimeStampedModel):
     doc_display = models.CharField(
         _('Doc Display'),
         max_length=4092,
+        null=True,
     )
     anchor = models.CharField(
         _('Anchor'),

readthedocs/templates/search/elastic_search.html

@@ -13,11 +13,6 @@
 
 {% endblock %}
 
-{% block extra_scripts %}
-<script type="text/javascript" src="{% static 'search/readthedocs-client.js' %}"></script>
-<!--<script type="text/javascript" src="{% static 'search/search-embed.js' %}"></script>-->
-{% endblock %}
-
 {% block project_editing %}
   {% if project_obj %}
     {% with search_active="active" project=project_obj %}
@@ -31,10 +26,12 @@
 <div class="navigable">
   <ul>
       <h5>{% trans 'Object Type' %}</h5>
-      <li class="{% if type == 'all' %}active{% endif %}"><a href="?{% url_replace request 'type' 'all' %}">{% trans 'All' %}</a></li>
+      {% if not project_obj %}
       <li class="{% if type == 'project' %}active{% endif %}"><a href="?{% url_replace request 'type' 'project' %}">{% trans 'Projects' %}</a></li>
+      {% endif %}
       <li class="{% if type == 'file' %}active{% endif %}"><a href="?{% url_replace request 'type' 'file' %}">{% trans 'Files' %}</a></li>
       <li class="{% if type == 'domain' %}active{% endif %}"><a href="?{% url_replace request 'type' 'domain' %}">{% trans 'Code API' %}</a></li>
+      {# <li class="{% if type == 'all' %}active{% endif %}"><a href="?{% url_replace request 'type' 'all' %}">{% trans 'All' %}</a></li> #}
 
       <hr>
 
@@ -195,7 +192,7 @@ <h3>
                       {% elif 'domain' in result.meta.index %}
 
                         <a href="{% doc_url result.project|get_project result.version result.doc_name %}?highlight={{ query }}#{{ result.anchor }}">
-                          {{ result.project }} - {{ result.name }} 
+                          {{ result.project }} - {% if result.meta.highlight.name|length %} {{ result.meta.highlight.name.0|safe }} {% else %} {{ result.name }} {% endif %}
                         </a>
                         <p class="fragment">
                         {{ result.type_display|capfirst }} in {{ result.doc_display }}