Fix logic involving creation of Sphinx Domains

readthedocs/projects/tasks.py

@@ -1396,10 +1396,34 @@ def warn(self, msg):
                     f'domain->name',
                 )
                 continue
+
+            # HACK: This is done because the difference between
+            # ``sphinx.builders.html.StandaloneHTMLBuilder``
+            # and ``sphinx.builders.dirhtml.DirectoryHTMLBuilder``.
+            # They both have different ways of generating HTML Files,
+            # and therefore the doc_name generated is different.
+            # More info on: http://www.sphinx-doc.org/en/master/usage/builders/index.html#builders
+            # Also see issue: https://github.com/readthedocs/readthedocs.org/issues/5821
+            if doc_name.endswith('/'):
+                doc_name += 'index.html'
+
             html_file = HTMLFile.objects.filter(
                 project=version.project, version=version,
                 path=doc_name, build=build,
             ).first()
+
+            if not html_file:
+                log.debug('[%s] [%s] [Build: %s] HTMLFile object not found. File: %s' % (
+                    version.project,
+                    version,
+                    build,
+                    doc_name,
+                ))
+
+                # Don't create Sphinx Domain objects
+                # if the HTMLFile object is not found.
+                continue
+
             SphinxDomain.objects.create(
                 project=version.project,
                 version=version,

readthedocs/search/documents.py

@@ -111,23 +111,40 @@ class Meta:
 
     def prepare_domains(self, html_file):
         """Prepares and returns the values for domains field."""
-        domains_qs = html_file.sphinx_domains.exclude(
-            domain='std',
-            type__in=['doc', 'label']
-        ).iterator()
-
-        all_domains = [
-            {
-                'role_name': domain.role_name,
-                'doc_name': domain.doc_name,
-                'anchor': domain.anchor,
-                'type_display': domain.type_display,
-                'doc_display': domain.doc_display,
-                'name': domain.name,
-                'display_name': domain.display_name if domain.display_name != '-' else '',
-            }
-            for domain in domains_qs
-        ]
+        all_domains = []
+
+        try:
+            domains_qs = html_file.sphinx_domains.exclude(
+                domain='std',
+                type__in=['doc', 'label']
+            ).iterator()
+
+            all_domains = [
+                {
+                    'role_name': domain.role_name,
+                    'doc_name': domain.doc_name,
+                    'anchor': domain.anchor,
+                    'type_display': domain.type_display,
+                    'doc_display': domain.doc_display,
+                    'name': domain.name,
+                    'display_name': domain.display_name if domain.display_name != '-' else '',
+                }
+                for domain in domains_qs
+            ]
+
+            log.debug("[%s] [%s] Domains for file %s are: %s" % (
+                html_file.project.slug,
+                html_file.version.slug,
+                html_file.path,
+                all_domains,
+            ))
+
+        except Exception:
+            log.exception("[%s] [%s] Error preparing domain data for file %s" % (
+                html_file.project.slug,
+                html_file.version.slug,
+                html_file.path,
+            ))
 
         return all_domains