Generate json artifacts

readthedocs_ext/readthedocs.py

@@ -3,26 +3,58 @@
 from __future__ import absolute_import
 
 import codecs
+import json
 import os
 import re
 import types
 from distutils.version import LooseVersion
 
 import sphinx
 from sphinx import package_dir
-from sphinx.builders.html import StandaloneHTMLBuilder, DirectoryHTMLBuilder, SingleFileHTMLBuilder
+from sphinx.builders.html import (DirectoryHTMLBuilder, SingleFileHTMLBuilder,
+                                  StandaloneHTMLBuilder)
 from sphinx.util.console import bold
 
 from .embed import EmbedDirective
 from .mixins import BuilderMixin
 
+try:
+    # Avaliable from Sphinx 1.6
+    from sphinx.util.logging import getLogger
+except ImportError:
+    from logging import getLogger
+
+log = getLogger(__name__)
+
 MEDIA_MAPPING = {
     "_static/jquery.js": "%sjavascript/jquery/jquery-2.0.3.min.js",
     "_static/underscore.js": "%sjavascript/underscore.js",
     "_static/doctools.js": "%sjavascript/doctools.js",
 }
 
 
+# Whitelist keys that we want to output
+# to the json artifacts.
+KEYS = [
+    'body',
+    'alabaster_version',
+    'display_toc',
+    'title',
+    'sourcename',
+    'customsidebar',
+    'metatags',
+    'current_page_name',
+    'next',
+    'rellinks',
+    'meta',
+    'parents',
+    'sidebars',
+    'toc',
+    'prev',
+    'page_source_suffix',
+]
+
+
 def finalize_media(app):
     """ Point media files at our media server. """
 
@@ -126,6 +158,36 @@ def rtd_render(self, template, render_context):
                                                         app.builder.templates)
 
 
+def generate_json_artifacts(app, pagename, templatename, context, doctree):
+    """
+    Generate JSON artifacts for each page.
+
+    This way we can skip generating this in other build step.
+    """
+    try:
+        # We need to get the output directory where the docs are built
+        # _build/html/_json.
+        build_json = os.path.abspath(
+            os.path.join(app.outdir, '_json')
+        )
+        outjson = os.path.join(build_json, pagename + '.fjson')
+        outdir = os.path.dirname(outjson)
+        if not os.path.exists(outdir):
+            os.makedirs(outdir)
+        with open(outjson, 'w+') as json_file:
+            to_context = {
+                key: context[key]
+                for key in KEYS
+                if key in context
+            }
+            json_file.write(json.dumps(to_context, indent=4))
+            log.info('{page} processed.'.format(page=outjson))
+    except Exception:
+        log.exception(
+            'Failure in JSON search dump for {page}'.format(page=outjson)
+        )
+
+
 class HtmlBuilderMixin(BuilderMixin):
 
     static_readthedocs_files = [
@@ -213,6 +275,7 @@ def setup(app):
     app.add_builder(ReadtheDocsSingleFileHTMLBuilderLocalMedia)
     app.connect('builder-inited', finalize_media)
     app.connect('html-page-context', update_body)
+    app.connect('html-page-context', generate_json_artifacts)
 
     # Embed
     app.add_directive('readthedocs-embed', EmbedDirective)

tests/test_integration.py

@@ -9,7 +9,12 @@ class LanguageIntegrationTests(unittest.TestCase):
 
     def _run_test(self, test_dir, test_file, test_string, builder='html'):
         with build_output(test_dir, test_file, builder) as data:
-            self.assertIn(test_string, data)
+            if not isinstance(test_string, list):
+                test_strings = [test_string]
+            else:
+                test_strings = test_string
+            for string in test_strings:
+                self.assertIn(string, data)
 
 
 class IntegrationTests(LanguageIntegrationTests):
@@ -56,3 +61,10 @@ def test_searchtools_is_patched(self):
                 HtmlBuilderMixin.REPLACEMENT_PATTERN
             )
             self.assertIn(HtmlBuilderMixin.REPLACEMENT_TEXT, data)
+
+    def test_generate_json_artifacts(self):
+        self._run_test(
+            'pyexample',
+            '_build/html/_json/index.fjson',
+            ['current_page_name', 'title', 'body', 'toc'],
+        )

tox.ini

@@ -1,6 +1,6 @@
 [tox]
 envlist =
-    py{27,35,36}-sphinx{13,14,15,16}
+    py{27,35,36}-sphinx{13,14,15,16,17}
     lint
 
 [testenv]
@@ -13,7 +13,8 @@ deps =
     sphinx13: Sphinx<1.4
     sphinx14: Sphinx<1.5
     sphinx15: Sphinx<1.6
-    sphinx16: Sphinx==1.6b3
+    sphinx16: Sphinx<1.7
+    sphinx17: Sphinx==1.7.4
 commands =
     py.test {posargs}