Use the embed API to show search results inline

readthedocs/domaindata/admin.py

@@ -0,0 +1,11 @@
+from django.contrib import admin
+from .models import DomainData
+
+
+class DomainDataAdmin(admin.ModelAdmin):
+    list_filter = ('type', 'project')
+    raw_id_fields = ('project', 'version')
+    search_fields = ('doc_name', 'name')
+
+
+admin.site.register(DomainData, DomainDataAdmin)

readthedocs/domaindata/api.py

@@ -0,0 +1,26 @@
+from rest_framework import serializers
+
+from readthedocs.restapi.views.model_views import UserSelectViewSet
+from .models import DomainData
+
+
+class DomainDataSerializer(serializers.ModelSerializer):
+    project = serializers.SlugRelatedField(slug_field='slug', read_only=True)
+    version = serializers.SlugRelatedField(slug_field='slug', read_only=True)
+
+    class Meta:
+        model = DomainData
+        fields = ('project', 'version', 'name', 'display_name', 'doc_type', 'doc_url')
+
+
+class DomainDataAdminSerializer(DomainDataSerializer):
+
+    class Meta(DomainDataSerializer.Meta):
+        fields = '__all__'
+
+
+class DomainDataAPIView(UserSelectViewSet):
+    model = DomainData
+    serializer_class = DomainDataSerializer
+    admin_serializer_class = DomainDataAdminSerializer
+    filter_fields = ('project__slug', 'version__slug', 'domain', 'type', 'doc_name', 'name')

readthedocs/domaindata/models.py

@@ -0,0 +1,72 @@
+from django.db import models
+from django.utils.encoding import python_2_unicode_compatible
+from django.utils.translation import ugettext_lazy as _
+
+from readthedocs.builds.models import Version
+from readthedocs.core.resolver import resolve
+from readthedocs.projects.models import Project
+from readthedocs.projects.querysets import RelatedProjectQuerySet
+
+
+@python_2_unicode_compatible
+class DomainData(models.Model):
+
+    """
+    Information from a project about it's Sphinx domains.
+
+    This captures data about API objects that exist in that codebase.
+    """
+
+    project = models.ForeignKey(
+        Project,
+        related_name='domain_data',
+    )
+    version = models.ForeignKey(Version, verbose_name=_('Version'),
+                                related_name='domain_data')
+    modified_date = models.DateTimeField(_('Publication date'), auto_now=True)
+    commit = models.CharField(_('Commit'), max_length=255)
+
+    domain = models.CharField(
+        _('Domain'),
+        max_length=255,
+    )
+    name = models.CharField(
+        _('Name'),
+        max_length=255,
+    )
+    display_name = models.CharField(
+        _('Display Name'),
+        max_length=255,
+    )
+    type = models.CharField(
+        _('Type'),
+        max_length=255,
+    )
+    doc_name = models.CharField(
+        _('Doc Name'),
+        max_length=255,
+    )
+    anchor = models.CharField(
+        _('Anchor'),
+        max_length=255,
+    )
+    objects = RelatedProjectQuerySet.as_manager()
+
+    def __str__(self):
+        return f'''
+            DomainData [{self.project.slug}:{self.version.slug}]
+            [{self.domain}:{self.type}] {self.name} -> {self.doc_name}#{self.anchor}
+            '''
+
+    @property
+    def doc_type(self):
+        return f'{self.domain}:{self.type}'
+
+    @property
+    def doc_url(self):
+        path = self.doc_name
+        if self.anchor:
+            path += f'#{self.anchor}'
+        full_url = resolve(
+            project=self.project, version_slug=self.version.slug, filename=path)
+        return full_url

readthedocs/projects/tasks.py

@@ -13,6 +13,7 @@
 import json
 import logging
 import os
+import sys
 import shutil
 import socket
 from collections import Counter, defaultdict
@@ -25,6 +26,8 @@
 from django.utils import timezone
 from django.utils.translation import ugettext_lazy as _
 from slumber.exceptions import HttpClientError
+from sphinx.ext import intersphinx
+
 
 from readthedocs.builds.constants import (
     BUILD_STATE_BUILDING,
@@ -58,6 +61,7 @@
 )
 from readthedocs.doc_builder.loader import get_builder_class
 from readthedocs.doc_builder.python_environments import Conda, Virtualenv
+from readthedocs.domaindata.models import DomainData
 from readthedocs.projects.models import APIProject
 from readthedocs.restapi.client import api as api_v2
 from readthedocs.vcs_support import utils as vcs_support_utils
@@ -1136,6 +1140,7 @@ def fileify(version_pk, commit):
             ),
         )
         _manage_imported_files(version, path, commit)
+        _update_intersphinx_data(version, path, commit)
     else:
         log.info(
             LOG_TEMPLATE.format(
@@ -1146,6 +1151,56 @@ def fileify(version_pk, commit):
         )
 
 
+def _update_intersphinx_data(version, path, commit):
+    """
+    Update intersphinx data for this version
+
+    :param version: Version instance
+    :param path: Path to search
+    :param commit: Commit that updated path
+    """
+    object_file = os.path.join(path, 'objects.inv')
+
+    class MockConfig:
+        intersphinx_timeout = None  # type: int
+        tls_verify = False
+
+    class MockApp:
+        srcdir = ''
+        config = MockConfig()
+
+        def warn(self, msg):
+            # type: (unicode) -> None
+            print(msg, file=sys.stderr)
+
+    invdata = intersphinx.fetch_inventory(MockApp(), '', object_file)
+    for key in sorted(invdata or {}):
+        domain, _type = key.split(':')
+        for name, einfo in sorted(invdata[key].items()):
+            url = einfo[2]
+            if '#' in url:
+                doc_name, anchor = url.split('#')
+            else:
+                doc_name, anchor = url, ''
+            display_name = einfo[3]
+            obj, _ = DomainData.objects.get_or_create(
+                project=version.project,
+                version=version,
+                domain=domain,
+                name=name,
+                display_name=display_name,
+                type=_type,
+                doc_name=doc_name,
+                anchor=anchor,
+            )
+            if obj.commit != commit:
+                obj.commit = commit
+                obj.save()
+    DomainData.objects.filter(project=version.project,
+                              version=version
+                              ).exclude(commit=commit).delete()
+
+
 def _manage_imported_files(version, path, commit):
     """
     Update imported files for version.

readthedocs/projects/urls/public.py

@@ -51,7 +51,7 @@
     ),
     url(
         r'^(?P<project_slug>{project_slug})/search/$'.format(**pattern_opts),
-        search_views.elastic_project_search,
+        search_views.elastic_search,
         name='elastic_project_search',
     ),
     url(

readthedocs/restapi/urls.py

@@ -25,6 +25,7 @@
     SocialAccountViewSet,
     VersionViewSet,
 )
+from readthedocs.domaindata.api import DomainDataAPIView
 
 
 router = routers.DefaultRouter()
@@ -34,6 +35,7 @@
 router.register(r'project', ProjectViewSet, basename='project')
 router.register(r'notification', NotificationViewSet, basename='emailhook')
 router.register(r'domain', DomainViewSet, basename='domain')
+router.register(r'domaindata', DomainDataAPIView, base_name='domaindata')
 router.register(
     r'remote/org',
     RemoteOrganizationViewSet,

readthedocs/search/documents.py

@@ -4,8 +4,8 @@
 from django.conf import settings
 from django_elasticsearch_dsl import DocType, Index, fields
 
-from readthedocs.projects.models import HTMLFile, Project
-
+from readthedocs.projects.models import Project, HTMLFile
+from readthedocs.domaindata.models import DomainData
 
 project_conf = settings.ES_INDEXES['project']
 project_index = Index(project_conf['name'])
@@ -15,9 +15,52 @@
 page_index = Index(page_conf['name'])
 page_index.settings(**page_conf['settings'])
 
+domain_conf = settings.ES_INDEXES['domain']
+domain_index = Index(domain_conf['name'])
+domain_index.settings(**domain_conf['settings'])
+
 log = logging.getLogger(__name__)
 
 
+@domain_index.doc_type
+class DomainDocument(DocType):
+    project = fields.KeywordField(attr='project.slug')
+    version = fields.KeywordField(attr='version.slug')
+    doc_type = fields.KeywordField(attr='doc_type')
+    anchor = fields.KeywordField(attr='anchor')
+
+    class Meta(object):
+        model = DomainData
+        fields = ('name', 'display_name', 'doc_name')
+        ignore_signals = True
+
+    @classmethod
+    def faceted_search(cls, query, user, doc_type=None):
+        from readthedocs.search.faceted_search import DomainSearch
+        kwargs = {
+            'user': user,
+            'query': query,
+        }
+
+        if doc_type:
+            kwargs['filters'] = {'doc_type': doc_type}
+
+        return DomainSearch(**kwargs)
+
+    def get_queryset(self):
+        """Overwrite default queryset to filter certain files to index"""
+        queryset = super().get_queryset()
+
+        # Exclude some types to not index
+        excluded_types = ['std:doc', 'std:label']
+
+        # Do not index files that belong to non sphinx project
+        # Also do not index certain files
+        for exclude in excluded_types:
+            queryset = queryset.exclude(type=exclude)
+        return queryset
+
+
 @project_index.doc_type
 class ProjectDocument(DocType):