Fix logic involving creation of Sphinx Domains

readthedocs/projects/tasks.py

@@ -1396,10 +1396,34 @@ def warn(self, msg):
                     f'domain->name',
                 )
                 continue
+
+            # HACK: This is done because the difference between
+            # ``sphinx.builders.html.StandaloneHTMLBuilder``
+            # and ``sphinx.builders.dirhtml.DirectoryHTMLBuilder``.
+            # They both have different ways of generating HTML Files,
+            # and therefore the doc_name generated is different.
+            # More info on: http://www.sphinx-doc.org/en/master/usage/builders/index.html#builders
+            # Also see issue: https://github.com/readthedocs/readthedocs.org/issues/5821
+            if doc_name.endswith('/'):
+                doc_name += 'index.html'
+
             html_file = HTMLFile.objects.filter(
                 project=version.project, version=version,
                 path=doc_name, build=build,
             ).first()
+
+            if not html_file:
+                log.debug('[%s] [%s] [Build: %s] HTMLFile object not found. File: %s' % (
+                    version.project,
+                    version,
+                    build,
+                    doc_name,
+                ))
+
+                # Don't create Sphinx Domain objects
+                # if the HTMLFile object is not found.
+                continue
+
             SphinxDomain.objects.create(
                 project=version.project,
                 version=version,

readthedocs/rtd_tests/files/api/index.html

@@ -0,0 +1 @@
+Test

readthedocs/rtd_tests/tests/test_imported_file.py

@@ -1,10 +1,17 @@
 # -*- coding: utf-8 -*-
+
 import os
+import mock
 
 from django.test import TestCase
 
-from readthedocs.projects.models import ImportedFile, Project
-from readthedocs.projects.tasks import _create_imported_files, _sync_imported_files
+from readthedocs.projects.models import ImportedFile, Project, HTMLFile
+from readthedocs.projects.tasks import (
+    _create_imported_files,
+    _sync_imported_files,
+    _create_intersphinx_data,
+)
+from readthedocs.sphinx_domains.models import SphinxDomain
 
 
 base_dir = os.path.dirname(os.path.dirname(__file__))
@@ -16,7 +23,7 @@ class ImportedFileTests(TestCase):
     def setUp(self):
         self.project = Project.objects.get(slug='pip')
         self.version = self.project.versions.first()
-    
+
     def _manage_imported_files(self, version, path, commit, build):
         """Helper function for the tests to create and sync ImportedFiles."""
         _create_imported_files(version, path, commit, build)
@@ -26,9 +33,9 @@ def test_properly_created(self):
         test_dir = os.path.join(base_dir, 'files')
         self.assertEqual(ImportedFile.objects.count(), 0)
         self._manage_imported_files(self.version, test_dir, 'commit01', 1)
-        self.assertEqual(ImportedFile.objects.count(), 3)
+        self.assertEqual(ImportedFile.objects.count(), 4)
         self._manage_imported_files(self.version, test_dir, 'commit01', 2)
-        self.assertEqual(ImportedFile.objects.count(), 3)
+        self.assertEqual(ImportedFile.objects.count(), 4)
 
     def test_update_commit(self):
         test_dir = os.path.join(base_dir, 'files')
@@ -54,4 +61,68 @@ def test_update_content(self):
         self._manage_imported_files(self.version, test_dir, 'commit02', 2)
         self.assertNotEqual(ImportedFile.objects.get(name='test.html').md5, 'c7532f22a052d716f7b2310fb52ad981')
 
-        self.assertEqual(ImportedFile.objects.count(), 3)
+        self.assertEqual(ImportedFile.objects.count(), 4)
+
+    @mock.patch('readthedocs.projects.tasks.os.path.exists')
+    def test_create_intersphinx_data(self, mock_exists):
+        mock_exists.return_Value = True
+
+        # Test data for objects.inv file
+        test_objects_inv = {
+            'cpp:function': {
+                'sphinx.test.function': [
+                    'dummy-proj-1',
+                    'dummy-version-1',
+                    'test.html#epub-faq',  # file generated by ``sphinx.builders.html.StandaloneHTMLBuilder``
+                    'dummy-func-name-1',
+                ]
+            },
+            'py:function': {
+                'sample.test.function': [
+                    'dummy-proj-2',
+                    'dummy-version-2',
+                    'test.html#sample-test-func',  # file generated by ``sphinx.builders.html.StandaloneHTMLBuilder``
+                    'dummy-func-name-2'
+                ]
+            },
+            'js:function': {
+                'testFunction': [
+                    'dummy-proj-3',
+                    'dummy-version-3',
+                    'api/#test-func',  # file generated by ``sphinx.builders.dirhtml.DirectoryHTMLBuilder``
+                    'dummy-func-name-3'
+                ]
+            }
+        }
+
+        with mock.patch(
+            'sphinx.ext.intersphinx.fetch_inventory',
+            return_value=test_objects_inv
+        ) as mock_fetch_inventory:
+
+            test_dir = os.path.join(base_dir, 'files')
+
+            _create_imported_files(self.version, test_dir, 'commit01', 1)
+            _create_intersphinx_data(self.version, test_dir, 'commit01', 1)
+
+            # there will be two html files,
+            # `api/index.html` and `test.html`
+            self.assertEqual(
+                HTMLFile.objects.all().count(),
+                2
+            )
+            self.assertEqual(
+                HTMLFile.objects.filter(path='api/index.html').count(),
+                1
+            )
+
+            html_file_api = HTMLFile.objects.filter(path='api/index.html').first()
+
+            self.assertEqual(
+                SphinxDomain.objects.all().count(),
+                3
+            )
+            self.assertEqual(
+                SphinxDomain.objects.filter(html_file=html_file_api).count(),
+                1
+            )

readthedocs/search/documents.py

@@ -111,23 +111,40 @@ class Meta:
 
     def prepare_domains(self, html_file):
         """Prepares and returns the values for domains field."""
-        domains_qs = html_file.sphinx_domains.exclude(
-            domain='std',
-            type__in=['doc', 'label']
-        ).iterator()
-
-        all_domains = [
-            {
-                'role_name': domain.role_name,
-                'doc_name': domain.doc_name,
-                'anchor': domain.anchor,
-                'type_display': domain.type_display,
-                'doc_display': domain.doc_display,
-                'name': domain.name,
-                'display_name': domain.display_name if domain.display_name != '-' else '',
-            }
-            for domain in domains_qs
-        ]
+        all_domains = []
+
+        try:
+            domains_qs = html_file.sphinx_domains.exclude(
+                domain='std',
+                type__in=['doc', 'label']
+            ).iterator()
+
+            all_domains = [
+                {
+                    'role_name': domain.role_name,
+                    'doc_name': domain.doc_name,
+                    'anchor': domain.anchor,
+                    'type_display': domain.type_display,
+                    'doc_display': domain.doc_display,
+                    'name': domain.name,
+                    'display_name': domain.display_name if domain.display_name != '-' else '',
+                }
+                for domain in domains_qs
+            ]
+
+            log.debug("[%s] [%s] Total domains for file %s are: %s" % (
+                html_file.project.slug,
+                html_file.version.slug,
+                html_file.path,
+                len(all_domains),
+            ))
+
+        except Exception:
+            log.exception("[%s] [%s] Error preparing domain data for file %s" % (
+                html_file.project.slug,
+                html_file.version.slug,
+                html_file.path,
+            ))
 
         return all_domains