Merge pull request #62 from readthedocs/get-domain-data

Increase the amount of data we're saving during build

Author: ericholscher

.travis.yml

@@ -19,3 +19,6 @@ notifications:
       - readthedocs:y3hjODOi7EIz1JAbD1Zb41sz#random
     on_success: change
     on_failure: always
+branches:
+  only:
+  - master

README.rst

@@ -23,7 +23,7 @@ Releasing
 
 #. Increment the version in ``setup.py``
 #. Tag the release in git: ``git tag $NEW_VERSION``.
-#. Push the tag to GitHub: ``git push --tags origin``
+#. Push the tag to GitHub: ``git push --tags origin master``
 #. Upload the package to PyPI:
 
     .. code:: bash

readthedocs_ext/readthedocs.py

@@ -15,6 +15,7 @@
                                   StandaloneHTMLBuilder)
 from sphinx.util.console import bold
 
+
 from .embed import EmbedDirective
 from .mixins import BuilderMixin
 
@@ -27,10 +28,20 @@
 log = getLogger(__name__)
 
 DEFAULT_STATIC_URL = 'https://assets.readthedocs.org/static/'
+ONLINE_BUILDERS = [
+    'readthedocs', 'readthedocsdirhtml', 'readthedocssinglehtml'
+]
+# Only run JSON output once during HTML build
+# This saves resources and keeps filepaths correct,
+# because singlehtml filepaths are different
+JSON_BUILDERS = [
+    'html', 'dirhtml',
+    'readthedocs', 'readthedocsdirhtml'
+]
 
 # Whitelist keys that we want to output
 # to the json artifacts.
-KEYS = [
+JSON_KEYS = [
     'body',
     'title',
     'sourcename',
@@ -65,15 +76,12 @@ def update_body(app, pagename, templatename, context, doctree):
     """
 
     STATIC_URL = context.get('STATIC_URL', DEFAULT_STATIC_URL)
-    online_builders = [
-        'readthedocs', 'readthedocsdirhtml', 'readthedocssinglehtml'
-    ]
     if app.builder.name == 'readthedocssinglehtmllocalmedia':
         if 'html_theme' in context and context['html_theme'] == 'sphinx_rtd_theme':
             theme_css = '_static/css/theme.css'
         else:
             theme_css = '_static/css/badge_only.css'
-    elif app.builder.name in online_builders:
+    elif app.builder.name in ONLINE_BUILDERS:
         if 'html_theme' in context and context['html_theme'] == 'sphinx_rtd_theme':
             theme_css = '%scss/sphinx_rtd_theme.css' % STATIC_URL
         else:
@@ -148,6 +156,8 @@ def generate_json_artifacts(app, pagename, templatename, context, doctree):
 
     This way we can skip generating this in other build step.
     """
+    if app.builder.name not in JSON_BUILDERS:
+        return
     try:
         # We need to get the output directory where the docs are built
         # _build/json.
@@ -161,7 +171,7 @@ def generate_json_artifacts(app, pagename, templatename, context, doctree):
         with open(outjson, 'w+') as json_file:
             to_context = {
                 key: context.get(key, '')
-                for key in KEYS
+                for key in JSON_KEYS
             }
             json.dump(to_context, json_file, indent=4)
     except TypeError:
@@ -172,12 +182,73 @@ def generate_json_artifacts(app, pagename, templatename, context, doctree):
         log.exception(
             'Fail to save JSON output for page {page}'.format(page=outjson)
         )
-    except Exception as e:
+    except Exception:
         log.exception(
             'Failure in JSON search dump for page {page}'.format(page=outjson)
         )
 
 
+def dump_sphinx_data(app, exception):
+    """
+    Dump data that is only in memory during Sphinx build.
+    This is mostly used for search indexing.
+
+    This includes:
+
+    * `paths`: A mapping of HTML Filename -> RST file
+    * `pages`: A mapping of HTML Filename -> Sphinx Page name
+    * `titles`: A mapping of HTML Filename -> Page Title
+    * `types`: A mapping of Sphinx Domain type slugs -> human-readable name for that type
+
+    """
+    if app.builder.name not in JSON_BUILDERS or exception:
+        return
+    try:
+        types = {}
+        titles = {}
+        paths = {}
+        pages = {}
+
+        for domain_name, domain_obj in app.env.domains.items():
+            for type_name, type_obj in domain_obj.object_types.items():
+                key = "{}:{}".format(domain_name, type_name)
+                types[key] = str(type_obj.lname)
+
+        for page, title in app.env.titles.items():
+            page_uri = app.builder.get_target_uri(page)
+            titles[page_uri] = title.astext()
+            paths[page_uri] = app.env.doc2path(page, base=None)
+            pages[page_uri] = page
+
+        to_dump = {
+            'types': types,
+            'titles': titles,
+            'paths': paths,
+            'pages': pages,
+        }
+
+        # We need to get the output directory where the docs are built
+        # _build/json.
+        build_json = os.path.abspath(
+            os.path.join(app.outdir, '..', 'json')
+        )
+        outjson = os.path.join(build_json, 'readthedocs-sphinx-domain-names.json')
+        with open(outjson, 'w+') as json_file:
+            json.dump(to_dump, json_file, indent=4)
+    except TypeError:
+        log.exception(
+            'Fail to encode JSON for object names'
+        )
+    except IOError:
+        log.exception(
+            'Fail to save JSON for object names'
+        )
+    except Exception:
+        log.exception(
+            'Failure in JSON search dump for object names'
+        )
+
+
 class HtmlBuilderMixin(BuilderMixin):
 
     static_readthedocs_files = [
@@ -271,6 +342,7 @@ def setup(app):
     app.connect('builder-inited', finalize_media)
     app.connect('html-page-context', update_body)
     app.connect('html-page-context', generate_json_artifacts)
+    app.connect('build-finished', dump_sphinx_data)
 
     # Embed
     app.add_directive('readthedocs-embed', EmbedDirective)

tests/test_integration.py

@@ -72,6 +72,22 @@ def test_generate_json_artifacts(self):
             ],
         )
 
+    def test_generate_json_domain_artifacts(self):
+        self._run_test(
+            'pyexample-json',
+            '_build/json/readthedocs-sphinx-domain-names.json',
+            [
+                # types
+                '"js:class": "class"',
+                # pages
+                '"index.html": "index"',
+                # paths
+                '"index.html": "index.rst"',
+                # titles
+                '"index.html": "Welcome to pyexample',
+            ],
+        )
+
     def test_escape_js_vars(self):
         with build_output('pyexample', '_build/readthedocs/escape\' this js.html',
                           builder='readthedocs') as data: