Add initial search implementation to DomainData

This move project search to a subset of site search.
This removes the second entry point to site search,
and unifies all our searching to use fancy facets and be nice.

Author: ericholscher

readthedocs/domaindata/__init__.py

@@ -0,0 +1,11 @@
+from django.contrib import admin
+from .models import DomainData
+
+
+class DomainDataAdmin(admin.ModelAdmin):
+    list_filter = ('type', 'project')
+    raw_id_fields = ('project', 'version')
+    search_fields = ('doc_name', 'name')
+
+
+admin.site.register(DomainData, DomainDataAdmin)

readthedocs/domaindata/admin.py

@@ -0,0 +1,26 @@
+from rest_framework import serializers
+
+from readthedocs.restapi.views.model_views import UserSelectViewSet
+from .models import DomainData
+
+
+class DomainDataSerializer(serializers.ModelSerializer):
+    project = serializers.SlugRelatedField(slug_field='slug', read_only=True)
+    version = serializers.SlugRelatedField(slug_field='slug', read_only=True)
+
+    class Meta:
+        model = DomainData
+        fields = ('project', 'version', 'name', 'display_name', 'doc_type', 'doc_url')
+
+
+class DomainDataAdminSerializer(DomainDataSerializer):
+
+    class Meta(DomainDataSerializer.Meta):
+        fields = '__all__'
+
+
+class DomainDataAPIView(UserSelectViewSet):
+    model = DomainData
+    serializer_class = DomainDataSerializer
+    admin_serializer_class = DomainDataAdminSerializer
+    filter_fields = ('project__slug', 'version__slug', 'domain', 'type', 'doc_name', 'name')

readthedocs/domaindata/api.py

@@ -0,0 +1,72 @@
+from django.db import models
+from django.utils.encoding import python_2_unicode_compatible
+from django.utils.translation import ugettext_lazy as _
+
+from readthedocs.builds.models import Version
+from readthedocs.core.resolver import resolve
+from readthedocs.projects.models import Project
+from readthedocs.projects.querysets import RelatedProjectQuerySet
+
+
+@python_2_unicode_compatible
+class DomainData(models.Model):
+
+    """
+    Information from a project about it's Sphinx domains.
+
+    This captures data about API objects that exist in that codebase.
+    """
+
+    project = models.ForeignKey(
+        Project,
+        related_name='domain_data',
+    )
+    version = models.ForeignKey(Version, verbose_name=_('Version'),
+                                related_name='domain_data')
+    modified_date = models.DateTimeField(_('Publication date'), auto_now=True)
+    commit = models.CharField(_('Commit'), max_length=255)
+
+    domain = models.CharField(
+        _('Domain'),
+        max_length=255,
+    )
+    name = models.CharField(
+        _('Name'),
+        max_length=255,
+    )
+    display_name = models.CharField(
+        _('Display Name'),
+        max_length=255,
+    )
+    type = models.CharField(
+        _('Type'),
+        max_length=255,
+    )
+    doc_name = models.CharField(
+        _('Doc Name'),
+        max_length=255,
+    )
+    anchor = models.CharField(
+        _('Anchor'),
+        max_length=255,
+    )
+    objects = RelatedProjectQuerySet.as_manager()
+
+    def __str__(self):
+        return f'''
+            DomainData [{self.project.slug}:{self.version.slug}]
+            [{self.domain}:{self.type}] {self.name} -> {self.doc_name}#{self.anchor}
+            '''
+
+    @property
+    def doc_type(self):
+        return f'{self.domain}:{self.type}'
+
+    @property
+    def doc_url(self):
+        path = self.doc_name
+        if self.anchor:
+            path += f'#{self.anchor}'
+        full_url = resolve(
+            project=self.project, version_slug=self.version.slug, filename=path)
+        return full_url

readthedocs/domaindata/models.py

@@ -11,6 +11,7 @@
 import json
 import logging
 import os
+import sys
 import shutil
 import socket
 from collections import Counter, defaultdict

readthedocs/projects/tasks.py

@@ -51,7 +51,7 @@
     ),
     url(
         r'^(?P<project_slug>{project_slug})/search/$'.format(**pattern_opts),
-        search_views.elastic_project_search,
+        search_views.elastic_search,
         name='elastic_project_search',
     ),
     url(

readthedocs/projects/urls/public.py

@@ -4,8 +4,8 @@
 from django.conf import settings
 from django_elasticsearch_dsl import DocType, Index, fields
 
-from readthedocs.projects.models import HTMLFile, Project
-
+from readthedocs.projects.models import Project, HTMLFile
+from readthedocs.domaindata.models import DomainData
 
 project_conf = settings.ES_INDEXES['project']
 project_index = Index(project_conf['name'])
@@ -15,9 +15,52 @@
 page_index = Index(page_conf['name'])
 page_index.settings(**page_conf['settings'])
 
+domain_conf = settings.ES_INDEXES['domain']
+domain_index = Index(domain_conf['name'])
+domain_index.settings(**domain_conf['settings'])
+
 log = logging.getLogger(__name__)
 
 
+@domain_index.doc_type
+class DomainDocument(DocType):
+    project = fields.KeywordField(attr='project.slug')
+    version = fields.KeywordField(attr='version.slug')
+    doc_type = fields.KeywordField(attr='doc_type')
+    anchor = fields.KeywordField(attr='anchor')
+
+    class Meta(object):
+        model = DomainData
+        fields = ('name', 'display_name', 'doc_name')
+        ignore_signals = True
+
+    @classmethod
+    def faceted_search(cls, query, user, doc_type=None):
+        from readthedocs.search.faceted_search import DomainSearch
+        kwargs = {
+            'user': user,
+            'query': query,
+        }
+
+        if doc_type:
+            kwargs['filters'] = {'doc_type': doc_type}
+
+        return DomainSearch(**kwargs)
+
+    def get_queryset(self):
+        """Overwrite default queryset to filter certain files to index"""
+        queryset = super().get_queryset()
+
+        # Exclude some types to not index
+        excluded_types = ['std:doc', 'std:label']
+
+        # Do not index files that belong to non sphinx project
+        # Also do not index certain files
+        for exclude in excluded_types:
+            queryset = queryset.exclude(type=exclude)
+        return queryset
+
+
 @project_index.doc_type
 class ProjectDocument(DocType):
 

readthedocs/search/documents.py

@@ -1,17 +1,40 @@
+# -*- coding: utf-8 -*-
 import logging
 
 from elasticsearch_dsl import FacetedSearch, TermsFacet
 from elasticsearch_dsl.query import Bool, SimpleQueryString
 
+from readthedocs.search.documents import (
+    DomainDocument,
+    PageDocument,
+    ProjectDocument,
+)
 from readthedocs.core.utils.extend import SettingsOverrideObject
-from readthedocs.search.documents import PageDocument, ProjectDocument
 
 log = logging.getLogger(__name__)
 
+ALL_FACETS = ['project', 'version', 'doc_type', 'language', 'index']
+
 
 class RTDFacetedSearch(FacetedSearch):
 
     def __init__(self, user, **kwargs):
+        self.user = user
+        self.filter_by_user = kwargs.pop('filter_by_user', None)
+
+        # Set filters properly
+        for facet in self.facets:
+            if facet in kwargs:
+                kwargs.setdefault('filters', {})[facet] = kwargs.pop(facet)
+
+        # Don't pass along unnecessary filters
+        for f in ALL_FACETS:
+            if f in kwargs:
+                del kwargs[f]
+
+        super().__init__(**kwargs)
+
+    def search(self):
         """
         Pass in a user in order to filter search results by privacy.
 
@@ -20,19 +43,36 @@ def __init__(self, user, **kwargs):
             The `self.user` attribute isn't currently used on the .org,
             but is used on the .com
         """
-        self.user = user
-        self.filter_by_user = kwargs.pop('filter_by_user', None)
-        super().__init__(**kwargs)
+        s = super().search()
+        s = s.source(exclude=['content', 'headers'])
+        # Return 25 results
+        return s[:25]
 
     def query(self, search, query):
         """
         Add query part to ``search`` when needed.
 
-        Also does HTML encoding of results to avoid XSS issues.
+        Also:
+
+        * Adds SimpleQueryString instead of default query.
+        * Adds HTML encoding of results to avoid XSS issues.
         """
-        search = super().query(search, query)
         search = search.highlight_options(encoder='html', number_of_fragments=3)
-        search = search.source(exclude=['content', 'headers'])
+
+        all_queries = []
+
+        # need to search for both 'and' and 'or' operations
+        # the score of and should be higher as it satisfies both or and and
+        for operator in ['and', 'or']:
+            query_string = SimpleQueryString(
+                query=query, fields=self.fields, default_operator=operator
+            )
+            all_queries.append(query_string)
+
+        # run bool query with should, so it returns result where either of the query matches
+        bool_query = Bool(should=all_queries)
+
+        search = search.query(bool_query)
         return search
 
 
@@ -52,26 +92,16 @@ class PageSearchBase(RTDFacetedSearch):
     index = PageDocument._doc_type.index
     fields = ['title^10', 'headers^5', 'content']
 
-    def query(self, search, query):
-        """Use a custom SimpleQueryString instead of default query."""
-
-        search = super().query(search, query)
-
-        all_queries = []
 
-        # need to search for both 'and' and 'or' operations
-        # the score of and should be higher as it satisfies both or and and
-        for operator in ['AND', 'OR']:
-            query_string = SimpleQueryString(
-                query=query, fields=self.fields, default_operator=operator
-            )
-            all_queries.append(query_string)
-
-        # run bool query with should, so it returns result where either of the query matches
-        bool_query = Bool(should=all_queries)
-
-        search = search.query(bool_query)
-        return search
+class DomainSearchBase(RTDFacetedSearch):
+    facets = {
+        'project': TermsFacet(field='project'),
+        'version': TermsFacet(field='version'),
+        'doc_type': TermsFacet(field='doc_type'),
+    }
+    doc_types = [DomainDocument]
+    index = DomainDocument._doc_type.index
+    fields = ('display_name^5', 'name')
 
 
 class PageSearch(SettingsOverrideObject):
@@ -94,3 +124,30 @@ class ProjectSearch(SettingsOverrideObject):
     """
 
     _default_class = ProjectSearchBase
+
+
+class DomainSearch(SettingsOverrideObject):
+
+    """
+    Allow this class to be overridden based on CLASS_OVERRIDES setting.
+
+    This is primary used on the .com to adjust how we filter our search queries
+    """
+
+    _default_class = DomainSearchBase
+
+
+class AllSearch(RTDFacetedSearch):
+    facets = {
+        'project': TermsFacet(field='project'),
+        'version': TermsFacet(field='version'),
+        'language': TermsFacet(field='language'),
+        'doc_type': TermsFacet(field='doc_type'),
+        'index': TermsFacet(field='_index'),
+    }
+    doc_types = [DomainDocument, PageDocument, ProjectDocument]
+    index = [DomainDocument._doc_type.index,
+             PageDocument._doc_type.index,
+             ProjectDocument._doc_type.index]
+    fields = ('title^10', 'headers^5', 'content', 'name^20',
+              'slug^5', 'description', 'display_name^5')