Add modeling for intersphinx data

readthedocs/domaindata/admin.py

@@ -0,0 +1,12 @@
+"""Domain Admin classes"""
+from django.contrib import admin
+from .models import DomainData
+
+
+class DomainDataAdmin(admin.ModelAdmin):
+    list_filter = ('type', 'project')
+    raw_id_fields = ('project', 'version')
+    search_fields = ('doc_name', 'name')
+
+
+admin.site.register(DomainData, DomainDataAdmin)

readthedocs/domaindata/api.py

@@ -0,0 +1,28 @@
+"""Domain API classes"""
+
+from rest_framework import serializers
+
+from readthedocs.restapi.views.model_views import UserSelectViewSet
+from .models import DomainData
+
+
+class DomainDataSerializer(serializers.ModelSerializer):
+    project = serializers.SlugRelatedField(slug_field='slug', read_only=True)
+    version = serializers.SlugRelatedField(slug_field='slug', read_only=True)
+
+    class Meta:
+        model = DomainData
+        fields = ('project', 'version', 'name', 'display_name', 'doc_type', 'doc_url')
+
+
+class DomainDataAdminSerializer(DomainDataSerializer):
+
+    class Meta(DomainDataSerializer.Meta):
+        fields = '__all__'
+
+
+class DomainDataAPIView(UserSelectViewSet):  # pylint: disable=too-many-ancestors
+    model = DomainData
+    serializer_class = DomainDataSerializer
+    admin_serializer_class = DomainDataAdminSerializer
+    filter_fields = ('project__slug', 'version__slug', 'domain', 'type', 'doc_name', 'name')

readthedocs/domaindata/models.py

@@ -0,0 +1,78 @@
+"""
+Sphinx Domain modeling.
+
+http://www.sphinx-doc.org/en/master/usage/restructuredtext/domains.html
+"""
+
+from django.db import models
+from django.utils.encoding import python_2_unicode_compatible
+from django.utils.translation import ugettext_lazy as _
+
+from readthedocs.builds.models import Version
+from readthedocs.core.resolver import resolve
+from readthedocs.projects.models import Project
+from readthedocs.projects.querysets import RelatedProjectQuerySet
+
+
+@python_2_unicode_compatible
+class DomainData(models.Model):
+
+    """
+    Information from a project about it's Sphinx domains.
+
+    This captures data about API objects that exist in that codebase.
+    """
+
+    project = models.ForeignKey(
+        Project,
+        related_name='domain_data',
+    )
+    version = models.ForeignKey(Version, verbose_name=_('Version'),
+                                related_name='domain_data')
+    modified_date = models.DateTimeField(_('Publication date'), auto_now=True)
+    commit = models.CharField(_('Commit'), max_length=255)
+
+    domain = models.CharField(
+        _('Domain'),
+        max_length=255,
+    )
+    name = models.CharField(
+        _('Name'),
+        max_length=255,
+    )
+    display_name = models.CharField(
+        _('Display Name'),
+        max_length=255,
+    )
+    type = models.CharField(
+        _('Type'),
+        max_length=255,
+    )
+    doc_name = models.CharField(
+        _('Doc Name'),
+        max_length=255,
+    )
+    anchor = models.CharField(
+        _('Anchor'),
+        max_length=255,
+    )
+    objects = RelatedProjectQuerySet.as_manager()
+
+    def __str__(self):
+        return f'''
+            DomainData [{self.project.slug}:{self.version.slug}]
+            [{self.domain}:{self.type}] {self.name} -> {self.doc_name}#{self.anchor}
+            '''
+
+    @property
+    def doc_type(self):
+        return f'{self.domain}:{self.type}'
+
+    @property
+    def doc_url(self):
+        path = self.doc_name
+        if self.anchor:
+            path += f'#{self.anchor}'
+        full_url = resolve(
+            project=self.project, version_slug=self.version.slug, filename=path)
+        return full_url

readthedocs/projects/tasks.py

@@ -13,6 +13,7 @@
 import json
 import logging
 import os
+import sys
 import shutil
 import socket
 from collections import Counter, defaultdict
@@ -25,6 +26,8 @@
 from django.utils import timezone
 from django.utils.translation import ugettext_lazy as _
 from slumber.exceptions import HttpClientError
+from sphinx.ext import intersphinx
+
 
 from readthedocs.builds.constants import (
     BUILD_STATE_BUILDING,
@@ -58,6 +61,7 @@
 )
 from readthedocs.doc_builder.loader import get_builder_class
 from readthedocs.doc_builder.python_environments import Conda, Virtualenv
+from readthedocs.domaindata.models import DomainData
 from readthedocs.projects.models import APIProject
 from readthedocs.restapi.client import api as api_v2
 from readthedocs.vcs_support import utils as vcs_support_utils
@@ -1136,6 +1140,7 @@ def fileify(version_pk, commit):
             ),
         )
         _manage_imported_files(version, path, commit)
+        _update_intersphinx_data(version, path, commit)
     else:
         log.info(
             LOG_TEMPLATE.format(
@@ -1146,6 +1151,56 @@ def fileify(version_pk, commit):
         )
 
 
+def _update_intersphinx_data(version, path, commit):
+    """
+    Update intersphinx data for this version
+
+    :param version: Version instance
+    :param path: Path to search
+    :param commit: Commit that updated path
+    """
+    object_file = os.path.join(path, 'objects.inv')
+
+    class MockConfig:
+        intersphinx_timeout = None  # type: int
+        tls_verify = False
+
+    class MockApp:
+        srcdir = ''
+        config = MockConfig()
+
+        def warn(self, msg):
+            # type: (unicode) -> None
+            print(msg, file=sys.stderr)
+
+    invdata = intersphinx.fetch_inventory(MockApp(), '', object_file)
+    for key in sorted(invdata or {}):
+        domain, _type = key.split(':')
+        for name, einfo in sorted(invdata[key].items()):
+            url = einfo[2]
+            if '#' in url:
+                doc_name, anchor = url.split('#')
+            else:
+                doc_name, anchor = url, ''
+            display_name = einfo[3]
+            obj, _ = DomainData.objects.get_or_create(
+                project=version.project,
+                version=version,
+                domain=domain,
+                name=name,
+                display_name=display_name,
+                type=_type,
+                doc_name=doc_name,
+                anchor=anchor,
+            )
+            if obj.commit != commit:
+                obj.commit = commit
+                obj.save()
+    DomainData.objects.filter(project=version.project,
+                              version=version
+                              ).exclude(commit=commit).delete()
+
+
 def _manage_imported_files(version, path, commit):
     """
     Update imported files for version.

readthedocs/restapi/urls.py

@@ -25,6 +25,7 @@
     SocialAccountViewSet,
     VersionViewSet,
 )
+from readthedocs.domaindata.api import DomainDataAPIView
 
 
 router = routers.DefaultRouter()
@@ -34,6 +35,7 @@
 router.register(r'project', ProjectViewSet, basename='project')
 router.register(r'notification', NotificationViewSet, basename='emailhook')
 router.register(r'domain', DomainViewSet, basename='domain')
+router.register(r'domaindata', DomainDataAPIView, base_name='domaindata')
 router.register(
     r'remote/org',
     RemoteOrganizationViewSet,

readthedocs/settings/base.py

@@ -100,6 +100,7 @@ def INSTALLED_APPS(self):  # noqa
             'readthedocs.notifications',
             'readthedocs.integrations',
             'readthedocs.analytics',
+            'readthedocs.domaindata',
             'readthedocs.search',