diff --git a/readthedocs/projects/tasks.py b/readthedocs/projects/tasks.py index 5106c9f7337..9e3fe12c603 100644 --- a/readthedocs/projects/tasks.py +++ b/readthedocs/projects/tasks.py @@ -1396,10 +1396,34 @@ def warn(self, msg): f'domain->name', ) continue + + # HACK: This is done because the difference between + # ``sphinx.builders.html.StandaloneHTMLBuilder`` + # and ``sphinx.builders.dirhtml.DirectoryHTMLBuilder``. + # They both have different ways of generating HTML Files, + # and therefore the doc_name generated is different. + # More info on: http://www.sphinx-doc.org/en/master/usage/builders/index.html#builders + # Also see issue: https://github.com/readthedocs/readthedocs.org/issues/5821 + if doc_name.endswith('/'): + doc_name += 'index.html' + html_file = HTMLFile.objects.filter( project=version.project, version=version, path=doc_name, build=build, ).first() + + if not html_file: + log.debug('[%s] [%s] [Build: %s] HTMLFile object not found. File: %s' % ( + version.project, + version, + build, + doc_name, + )) + + # Don't create Sphinx Domain objects + # if the HTMLFile object is not found. + continue + SphinxDomain.objects.create( project=version.project, version=version, diff --git a/readthedocs/rtd_tests/files/api/index.html b/readthedocs/rtd_tests/files/api/index.html new file mode 100644 index 00000000000..8318c86b357 --- /dev/null +++ b/readthedocs/rtd_tests/files/api/index.html @@ -0,0 +1 @@ +Test \ No newline at end of file diff --git a/readthedocs/rtd_tests/tests/test_imported_file.py b/readthedocs/rtd_tests/tests/test_imported_file.py index bbc1439ae88..184983b8e77 100644 --- a/readthedocs/rtd_tests/tests/test_imported_file.py +++ b/readthedocs/rtd_tests/tests/test_imported_file.py @@ -1,10 +1,17 @@ # -*- coding: utf-8 -*- + import os +import mock from django.test import TestCase -from readthedocs.projects.models import ImportedFile, Project -from readthedocs.projects.tasks import _create_imported_files, _sync_imported_files +from readthedocs.projects.models import ImportedFile, Project, HTMLFile +from readthedocs.projects.tasks import ( + _create_imported_files, + _sync_imported_files, + _create_intersphinx_data, +) +from readthedocs.sphinx_domains.models import SphinxDomain base_dir = os.path.dirname(os.path.dirname(__file__)) @@ -16,7 +23,7 @@ class ImportedFileTests(TestCase): def setUp(self): self.project = Project.objects.get(slug='pip') self.version = self.project.versions.first() - + def _manage_imported_files(self, version, path, commit, build): """Helper function for the tests to create and sync ImportedFiles.""" _create_imported_files(version, path, commit, build) @@ -26,9 +33,9 @@ def test_properly_created(self): test_dir = os.path.join(base_dir, 'files') self.assertEqual(ImportedFile.objects.count(), 0) self._manage_imported_files(self.version, test_dir, 'commit01', 1) - self.assertEqual(ImportedFile.objects.count(), 3) + self.assertEqual(ImportedFile.objects.count(), 4) self._manage_imported_files(self.version, test_dir, 'commit01', 2) - self.assertEqual(ImportedFile.objects.count(), 3) + self.assertEqual(ImportedFile.objects.count(), 4) def test_update_commit(self): test_dir = os.path.join(base_dir, 'files') @@ -54,4 +61,68 @@ def test_update_content(self): self._manage_imported_files(self.version, test_dir, 'commit02', 2) self.assertNotEqual(ImportedFile.objects.get(name='test.html').md5, 'c7532f22a052d716f7b2310fb52ad981') - self.assertEqual(ImportedFile.objects.count(), 3) + self.assertEqual(ImportedFile.objects.count(), 4) + + @mock.patch('readthedocs.projects.tasks.os.path.exists') + def test_create_intersphinx_data(self, mock_exists): + mock_exists.return_Value = True + + # Test data for objects.inv file + test_objects_inv = { + 'cpp:function': { + 'sphinx.test.function': [ + 'dummy-proj-1', + 'dummy-version-1', + 'test.html#epub-faq', # file generated by ``sphinx.builders.html.StandaloneHTMLBuilder`` + 'dummy-func-name-1', + ] + }, + 'py:function': { + 'sample.test.function': [ + 'dummy-proj-2', + 'dummy-version-2', + 'test.html#sample-test-func', # file generated by ``sphinx.builders.html.StandaloneHTMLBuilder`` + 'dummy-func-name-2' + ] + }, + 'js:function': { + 'testFunction': [ + 'dummy-proj-3', + 'dummy-version-3', + 'api/#test-func', # file generated by ``sphinx.builders.dirhtml.DirectoryHTMLBuilder`` + 'dummy-func-name-3' + ] + } + } + + with mock.patch( + 'sphinx.ext.intersphinx.fetch_inventory', + return_value=test_objects_inv + ) as mock_fetch_inventory: + + test_dir = os.path.join(base_dir, 'files') + + _create_imported_files(self.version, test_dir, 'commit01', 1) + _create_intersphinx_data(self.version, test_dir, 'commit01', 1) + + # there will be two html files, + # `api/index.html` and `test.html` + self.assertEqual( + HTMLFile.objects.all().count(), + 2 + ) + self.assertEqual( + HTMLFile.objects.filter(path='api/index.html').count(), + 1 + ) + + html_file_api = HTMLFile.objects.filter(path='api/index.html').first() + + self.assertEqual( + SphinxDomain.objects.all().count(), + 3 + ) + self.assertEqual( + SphinxDomain.objects.filter(html_file=html_file_api).count(), + 1 + ) diff --git a/readthedocs/search/documents.py b/readthedocs/search/documents.py index f1da48c1c17..04841e4cb40 100644 --- a/readthedocs/search/documents.py +++ b/readthedocs/search/documents.py @@ -111,23 +111,40 @@ class Meta: def prepare_domains(self, html_file): """Prepares and returns the values for domains field.""" - domains_qs = html_file.sphinx_domains.exclude( - domain='std', - type__in=['doc', 'label'] - ).iterator() - - all_domains = [ - { - 'role_name': domain.role_name, - 'doc_name': domain.doc_name, - 'anchor': domain.anchor, - 'type_display': domain.type_display, - 'doc_display': domain.doc_display, - 'name': domain.name, - 'display_name': domain.display_name if domain.display_name != '-' else '', - } - for domain in domains_qs - ] + all_domains = [] + + try: + domains_qs = html_file.sphinx_domains.exclude( + domain='std', + type__in=['doc', 'label'] + ).iterator() + + all_domains = [ + { + 'role_name': domain.role_name, + 'doc_name': domain.doc_name, + 'anchor': domain.anchor, + 'type_display': domain.type_display, + 'doc_display': domain.doc_display, + 'name': domain.name, + 'display_name': domain.display_name if domain.display_name != '-' else '', + } + for domain in domains_qs + ] + + log.debug("[%s] [%s] Total domains for file %s are: %s" % ( + html_file.project.slug, + html_file.version.slug, + html_file.path, + len(all_domains), + )) + + except Exception: + log.exception("[%s] [%s] Error preparing domain data for file %s" % ( + html_file.project.slug, + html_file.version.slug, + html_file.path, + )) return all_domains