Re-add Domain search

Author: ericholscher

readthedocs/search/documents.py

@@ -4,8 +4,8 @@
 from django.conf import settings
 from django_elasticsearch_dsl import DocType, Index, fields
 
-from readthedocs.projects.models import HTMLFile, Project
-
+from readthedocs.projects.models import Project, HTMLFile
+from readthedocs.domaindata.models import DomainData
 
 project_conf = settings.ES_INDEXES['project']
 project_index = Index(project_conf['name'])
@@ -15,9 +15,52 @@
 page_index = Index(page_conf['name'])
 page_index.settings(**page_conf['settings'])
 
+domain_conf = settings.ES_INDEXES['domain']
+domain_index = Index(domain_conf['name'])
+domain_index.settings(**domain_conf['settings'])
+
 log = logging.getLogger(__name__)
 
 
+@domain_index.doc_type
+class DomainDocument(DocType):
+    project = fields.KeywordField(attr='project.slug')
+    version = fields.KeywordField(attr='version.slug')
+    doc_type = fields.KeywordField(attr='doc_type')
+    anchor = fields.KeywordField(attr='anchor')
+
+    class Meta(object):
+        model = DomainData
+        fields = ('name', 'display_name', 'doc_name')
+        ignore_signals = True
+
+    @classmethod
+    def faceted_search(cls, query, user, doc_type=None):
+        from readthedocs.search.faceted_search import DomainSearch
+        kwargs = {
+            'user': user,
+            'query': query,
+        }
+
+        if doc_type:
+            kwargs['filters'] = {'doc_type': doc_type}
+
+        return DomainSearch(**kwargs)
+
+    def get_queryset(self):
+        """Overwrite default queryset to filter certain files to index"""
+        queryset = super().get_queryset()
+
+        # Exclude some types to not index
+        excluded_types = ['std:doc', 'std:label']
+
+        # Do not index files that belong to non sphinx project
+        # Also do not index certain files
+        for exclude in excluded_types:
+            queryset = queryset.exclude(type=exclude)
+        return queryset
+
+
 @project_index.doc_type
 class ProjectDocument(DocType):
 

readthedocs/search/faceted_search.py

@@ -1,17 +1,44 @@
+# -*- coding: utf-8 -*-
 import logging
 
 from elasticsearch_dsl import FacetedSearch, TermsFacet
 from elasticsearch_dsl.query import Bool, SimpleQueryString
 
+from readthedocs.search.documents import (
+    DomainDocument,
+    PageDocument,
+    ProjectDocument,
+)
+from readthedocs.search.signals import (
+    before_domain_search,
+    before_file_search,
+    before_project_search,
+)
+
 from readthedocs.core.utils.extend import SettingsOverrideObject
-from readthedocs.search.documents import PageDocument, ProjectDocument
 
 log = logging.getLogger(__name__)
 
 
+ALL_FACETS = ['project', 'version', 'doc_type', 'language', 'index']
+
+
 class RTDFacetedSearch(FacetedSearch):
 
     def __init__(self, user, **kwargs):
+        self.user = user
+        self.filter_by_user = kwargs.pop('filter_by_user', None)
+        for facet in self.facets:
+            if facet in kwargs:
+                kwargs.setdefault('filters', {})[facet] = kwargs.pop(facet)
+
+        # Don't pass along unnecessary filters
+        for f in ALL_FACETS:
+            if f in kwargs:
+                del kwargs[f]
+        super(RTDFacetedSearch, self).__init__(**kwargs)
+
+    def search(self):
         """
         Pass in a user in order to filter search results by privacy.
 
@@ -20,48 +47,36 @@ def __init__(self, user, **kwargs):
             The `self.user` attribute isn't currently used on the .org,
             but is used on the .com
         """
-        self.user = user
-        self.filter_by_user = kwargs.pop('filter_by_user', None)
-        super().__init__(**kwargs)
+        s = super().search()
+        s = s.source(exclude=['content', 'headers'])
+        resp = self.signal.send(sender=self, user=self.user, search=s)
+        if resp:
+            # Signal return a search object
+            try:
+                s = resp[0][1]
+            except AttributeError:
+                log.exception(
+                    'Failed to return a search object from search signals'
+                )
+        # Return 25 results
+        return s[:25]
 
     def query(self, search, query):
         """
         Add query part to ``search`` when needed.
 
-        Also does HTML encoding of results to avoid XSS issues.
+        Also:
+
+        * Adds SimpleQueryString instead of default query.
+        * Adds HTML encoding of results to avoid XSS issues.
         """
-        search = super().query(search, query)
         search = search.highlight_options(encoder='html', number_of_fragments=3)
-        search = search.source(exclude=['content', 'headers'])
-        return search
-
-
-class ProjectSearchBase(RTDFacetedSearch):
-    facets = {'language': TermsFacet(field='language')}
-    doc_types = [ProjectDocument]
-    index = ProjectDocument._doc_type.index
-    fields = ('name^10', 'slug^5', 'description')
-
-
-class PageSearchBase(RTDFacetedSearch):
-    facets = {
-        'project': TermsFacet(field='project'),
-        'version': TermsFacet(field='version')
-    }
-    doc_types = [PageDocument]
-    index = PageDocument._doc_type.index
-    fields = ['title^10', 'headers^5', 'content']
-
-    def query(self, search, query):
-        """Use a custom SimpleQueryString instead of default query."""
-
-        search = super().query(search, query)
 
         all_queries = []
 
         # need to search for both 'and' and 'or' operations
         # the score of and should be higher as it satisfies both or and and
-        for operator in ['AND', 'OR']:
+        for operator in ['and', 'or']:
             query_string = SimpleQueryString(
                 query=query, fields=self.fields, default_operator=operator
             )
@@ -74,23 +89,50 @@ def query(self, search, query):
         return search
 
 
-class PageSearch(SettingsOverrideObject):
-
-    """
-    Allow this class to be overridden based on CLASS_OVERRIDES setting.
-
-    This is primary used on the .com to adjust how we filter our search queries
-    """
+class DomainSearch(RTDFacetedSearch):
+    facets = {
+        'project': TermsFacet(field='project'),
+        'version': TermsFacet(field='version'),
+        'doc_type': TermsFacet(field='doc_type'),
+    }
+    signal = before_domain_search
+    doc_types = [DomainDocument]
+    index = DomainDocument._doc_type.index
+    fields = ('display_name^5', 'name')
 
-    _default_class = PageSearchBase
 
+class ProjectSearch(RTDFacetedSearch):
+    facets = {
+        'language': TermsFacet(field='language')
+    }
+    signal = before_project_search
+    doc_types = [ProjectDocument]
+    index = ProjectDocument._doc_type.index
+    fields = ('name^10', 'slug^5', 'description')
 
-class ProjectSearch(SettingsOverrideObject):
 
-    """
-    Allow this class to be overridden based on CLASS_OVERRIDES setting.
+class PageSearchBase(RTDFacetedSearch):
+    facets = {
+        'project': TermsFacet(field='project'),
+        'version': TermsFacet(field='version')
+    }
+    doc_types = [PageDocument]
+    index = PageDocument._doc_type.index
+    fields = ['title^10', 'headers^5', 'content']
 
-    This is primary used on the .com to adjust how we filter our search queries
-    """
 
-    _default_class = ProjectSearchBase
+class AllSearch(RTDFacetedSearch):
+    facets = {
+        'project': TermsFacet(field='project'),
+        'version': TermsFacet(field='version'),
+        'language': TermsFacet(field='language'),
+        'doc_type': TermsFacet(field='doc_type'),
+        'index': TermsFacet(field='_index'),
+    }
+    signal = before_file_search
+    doc_types = [DomainDocument, PageDocument, ProjectDocument]
+    index = [DomainDocument._doc_type.index,
+             PageDocument._doc_type.index,
+             ProjectDocument._doc_type.index]
+    fields = ('title^10', 'headers^5', 'content', 'name^20',
+              'slug^5', 'description', 'display_name^5')