DOC: Use nbsphinx

Update header levels for nbsphinx

Link to nb, nicer default table

Author: TomAugspurger

ci/requirements-3.5_DOC_BUILD.run

@@ -5,6 +5,7 @@ nbconvert
 nbformat
 notebook
 matplotlib
+seaborn
 scipy
 lxml
 beautifulsoup4

ci/requirements-3.5_DOC_BUILD.sh

@@ -6,6 +6,6 @@ echo "[install DOC_BUILD deps]"
 
 pip install pandas-gbq
 
-conda install -n pandas -c conda-forge feather-format
+conda install -n pandas -c conda-forge feather-format nbsphinx
 
 conda install -n pandas -c r r rpy2 --yes

ci/requirements_all.txt

@@ -3,6 +3,7 @@ pytest-cov
 pytest-xdist
 flake8
 sphinx
+nbsphinx
 ipython
 python-dateutil
 pytz

doc/README.rst

@@ -81,7 +81,9 @@ have ``sphinx`` and ``ipython`` installed. `numpydoc
 <https://github.com/numpy/numpydoc>`_ is used to parse the docstrings that
 follow the Numpy Docstring Standard (see above), but you don't need to install
 this because a local copy of ``numpydoc`` is included in the pandas source
-code.
+code. `nbsphinx <https://nbsphinx.readthedocs.io/>`_ is used to convert
+Jupyter notebooks. You will need to install it if you intend to modify any of
+the notebooks included in the documentation.
 
 Furthermore, it is recommended to have all `optional dependencies
 <http://pandas.pydata.org/pandas-docs/dev/install.html#optional-dependencies>`_

doc/make.py

@@ -106,106 +106,47 @@ def clean():
 
 
 @contextmanager
-def cleanup_nb(nb):
-    try:
-        yield
-    finally:
-        try:
-            os.remove(nb + '.executed')
-        except OSError:
-            pass
-
-
-def get_kernel():
-    """Find the kernel name for your python version"""
-    return 'python%s' % sys.version_info.major
-
-
-def execute_nb(src, dst, allow_errors=False, timeout=1000, kernel_name=''):
+def maybe_exclude_notebooks():
     """
-    Execute notebook in `src` and write the output to `dst`
-
-    Parameters
-    ----------
-    src, dst: str
-        path to notebook
-    allow_errors: bool
-    timeout: int
-    kernel_name: str
-        defualts to value set in notebook metadata
-
-    Returns
-    -------
-    dst: str
+    Skip building the notebooks if pandoc is not installed.
+    This assumes that nbsphinx is installed.
     """
-    import nbformat
-    from nbconvert.preprocessors import ExecutePreprocessor
-
-    with io.open(src, encoding='utf-8') as f:
-        nb = nbformat.read(f, as_version=4)
-
-    ep = ExecutePreprocessor(allow_errors=allow_errors,
-                             timeout=timeout,
-                             kernel_name=kernel_name)
-    ep.preprocess(nb, resources={})
-
-    with io.open(dst, 'wt', encoding='utf-8') as f:
-        nbformat.write(nb, f)
-    return dst
-
-
-def convert_nb(src, dst, to='html', template_file='basic'):
-    """
-    Convert a notebook `src`.
-
-    Parameters
-    ----------
-    src, dst: str
-        filepaths
-    to: {'rst', 'html'}
-        format to export to
-    template_file: str
-        name of template file to use. Default 'basic'
-    """
-    from nbconvert import HTMLExporter, RSTExporter
-
-    dispatch = {'rst': RSTExporter, 'html': HTMLExporter}
-    exporter = dispatch[to.lower()](template_file=template_file)
-
-    (body, resources) = exporter.from_filename(src)
-    with io.open(dst, 'wt', encoding='utf-8') as f:
-        f.write(body)
-    return dst
-
+    base = os.path.dirname(__file__)
+    notebooks = [os.path.join(base, 'source', nb)
+                 for nb in ['style.ipynb']]
+
+    checkpoints = os.listdir(
+        os.path.join(base, "source", ".ipynb_checkpoints")
+    )
+    notebooks.extend([os.path.join(base, 'source', '.ipynb_checkpoints', cp)
+                      for cp in checkpoints])
+    contents = {}
+    try:
+        import nbconvert
+        nbconvert.utils.pandoc.get_pandoc_version()
+    except (ImportError, nbconvert.utils.pandoc.PandocMissing):
+        print("Warning: Pandoc is not installed. Skipping Notebooks.")
+        for nb in notebooks:
+            with open(nb, 'rt') as f:
+                contents[nb] = f.read()
+            os.remove(nb)
+    yield
+    for nb, content in contents.items():
+        with open(nb, 'wt') as f:
+            f.write(content)
 
 def html():
     check_build()
 
-    notebooks = [
-        'source/html-styling.ipynb',
-    ]
-
-    for nb in notebooks:
-        with cleanup_nb(nb):
-            try:
-                print("Converting %s" % nb)
-                kernel_name = get_kernel()
-                executed = execute_nb(nb, nb + '.executed', allow_errors=True,
-                                      kernel_name=kernel_name)
-                convert_nb(executed, nb.rstrip('.ipynb') + '.html')
-            except (ImportError, IndexError) as e:
-                print(e)
-                print("Failed to convert %s" % nb)
-
-    if os.system('sphinx-build -j 2 -P -b html -d build/doctrees '
-                 'source build/html'):
-        raise SystemExit("Building HTML failed.")
-    try:
-        # remove stale file
-        os.remove('source/html-styling.html')
-        os.remove('build/html/pandas.zip')
-    except:
-        pass
+    with maybe_exclude_notebooks():
+        if os.system('sphinx-build -P -b html -d build/doctrees '
+                     'source build/html'):
+            raise SystemExit("Building HTML failed.")
+        try:
+            # remove stale file
+            os.remove('build/html/pandas.zip')
+        except:
+            pass
 
 
 def zip_html():

doc/source/conf.py

@@ -49,9 +49,10 @@
               'sphinx.ext.mathjax',
               'sphinx.ext.ifconfig',
               'sphinx.ext.linkcode',
+              'nbsphinx',
               ]
 
-
+exclude_patterns = ['**.ipynb_checkpoints']
 
 with open("index.rst") as f:
     index_rst_lines = f.readlines()
@@ -62,15 +63,16 @@
 # JP: added from sphinxdocs
 autosummary_generate = False
 
-if any([re.match("\s*api\s*",l) for l in index_rst_lines]):
+if any([re.match("\s*api\s*", l) for l in index_rst_lines]):
     autosummary_generate = True
 
 files_to_delete = []
 for f in os.listdir(os.path.dirname(__file__)):
-    if not f.endswith('.rst') or f.startswith('.') or os.path.basename(f) == 'index.rst':
+    if (not f.endswith(('.ipynb', '.rst')) or
+            f.startswith('.') or os.path.basename(f) == 'index.rst'):
         continue
 
-    _file_basename = f.split('.rst')[0]
+    _file_basename = os.path.splitext(f)[0]
     _regex_to_match = "\s*{}\s*$".format(_file_basename)
     if not any([re.match(_regex_to_match, line) for line in index_rst_lines]):
         files_to_delete.append(f)
@@ -253,6 +255,9 @@
 # Output file base name for HTML help builder.
 htmlhelp_basename = 'pandas'
 
+# -- Options for nbsphinx ------------------------------------------------
+
+nbsphinx_allow_errors = True
 
 # -- Options for LaTeX output --------------------------------------------
 

doc/source/contributing.rst

@@ -359,15 +359,14 @@ have ``sphinx`` and ``ipython`` installed. `numpydoc
 <https://github.com/numpy/numpydoc>`_ is used to parse the docstrings that
 follow the Numpy Docstring Standard (see above), but you don't need to install
 this because a local copy of numpydoc is included in the *pandas* source
-code.
-`nbconvert <https://nbconvert.readthedocs.io/en/latest/>`_ and
-`nbformat <https://nbformat.readthedocs.io/en/latest/>`_ are required to build
+code. `nbsphinx <https://nbsphinx.readthedocs.io/>`_ is required to build
 the Jupyter notebooks included in the documentation.
 
 If you have a conda environment named ``pandas_dev``, you can install the extra
 requirements with::
 
       conda install -n pandas_dev sphinx ipython nbconvert nbformat
+      conda install -n pandas_dev -c conda-forge nbsphinx
 
 Furthermore, it is recommended to have all :ref:`optional dependencies <install.optional_dependencies>`.
 installed. This is not strictly necessary, but be aware that you will see some error