ENH: Enable validation during sphinx-build process

doc/conf.py

@@ -84,6 +84,9 @@
 version = re.sub(r'(\.dev\d+).*?$', r'\1', version)
 numpydoc_xref_param_type = True
 numpydoc_xref_ignore = {'optional', 'type_without_description', 'BadException'}
+# Run docstring validation as part of build process
+numpydoc_validate = True
+numpydoc_validation_checks = {"all", "GL01", "SA04", "RT03"}
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

doc/install.rst

@@ -96,6 +96,31 @@ numpydoc_xref_ignore : set or ``"all"``
   desired cross reference mappings in ``numpydoc_xref_aliases`` and setting
   ``numpydoc_xref_ignore="all"`` is more convenient than explicitly listing
   terms to ignore in a set.
+numpydoc_validate : bool
+    Whether or not to run docstring validation during the sphinx-build process.
+    Default is ``False``.
+numpydoc_validation_checks : set
+    The set of validation checks to report during the sphinx build process.
+    Only has an effect when ``numpydoc_validate = True``.
+    If ``"all"`` is in the set, then the results of all of the
+    :ref:`built-in validation checks <validation_checks>` are reported.
+    If the set includes ``"all"`` and additional error codes, then all
+    validation checks *except* the listed error codes will be run.
+    If the set contains *only* individual error codes, then only those checks
+    will be run.
+    For example::
+
+        # Report warnings for all validation checks
+        numpydoc_validation_checks = {"all"}
+
+        # Report warnings for all checks *except* for GL01, GL02, and GL05
+        numpydoc_validation_checks = {"all", "GL01", "GL02", "GL05"}
+
+        # Only report warnings for the SA01 and EX01 checks
+        numpydoc_validation_checks = {"SA01", "EX01"}
+
+    The default is an empty set, thus no warnings from docstring validation
+    are reported.
 numpydoc_edit_link : bool
   .. deprecated:: 0.7.0
 

doc/validation.rst

@@ -15,3 +15,21 @@ For an exhaustive validation of the formatting of the docstring, use the
 ``--validate`` parameter. This will report the errors detected, such as
 incorrect capitalization, wrong order of the sections, and many other
 issues.
+
+.. _validation_checks:
+
+Built-in Validation Checks
+--------------------------
+
+The ``numpydoc.validation`` module provides a mapping with all of the checks
+that are run as part of the validation procedure.
+The mapping is of the form: ``error_code : <explanation>`` where ``error_code``
+provides a shorthand for the check being run, and ``<explanation>`` provides
+a more detailed message. For example::
+
+    "EX01" : "No examples section found"
+
+The full mapping of validation checks is given below.
+
+.. literalinclude:: ../numpydoc/validate.py
+   :lines: 36-90

numpydoc/__main__.py

@@ -6,13 +6,13 @@
 import ast
 
 from .docscrape_sphinx import get_doc_object
-from .validate import validate, Docstring
+from .validate import validate, Validator
 
 
 def render_object(import_path, config=None):
     """Test numpydoc docstring generation for a given object"""
-    # TODO: Move Docstring._load_obj to a better place than validate
-    print(get_doc_object(Docstring(import_path).obj,
+    # TODO: Move Validator._load_obj to a better place than validate
+    print(get_doc_object(Validator._load_obj(import_path),
                          config=dict(config or [])))
     return 0
 

numpydoc/docscrape.py

@@ -411,8 +411,16 @@ def _parse(self):
             else:
                 self[section] = content
 
+    @property
+    def _obj(self):
+        if hasattr(self, '_cls'):
+            return self._cls
+        elif hasattr(self, '_f'):
+            return self._f
+        return None
+
     def _error_location(self, msg, error=True):
-        if hasattr(self, '_obj') and self._obj is not None:
+        if self._obj is not None:
             # we know where the docs came from:
             try:
                 filename = inspect.getsourcefile(self._obj)
@@ -581,6 +589,12 @@ def __str__(self):
         return out
 
 
+class ObjDoc(NumpyDocString):
+    def __init__(self, obj, doc=None, config={}):
+        self._f = obj
+        NumpyDocString.__init__(self, doc, config=config)
+
+
 class ClassDoc(NumpyDocString):
 
     extra_public_methods = ['__call__']
@@ -663,3 +677,24 @@ def _is_show_member(self, name):
         if name not in self._cls.__dict__:
             return False  # class member is inherited, we do not show it
         return True
+
+
+def get_doc_object(obj, what=None, doc=None, config={}):
+    if what is None:
+        if inspect.isclass(obj):
+            what = 'class'
+        elif inspect.ismodule(obj):
+            what = 'module'
+        elif isinstance(obj, Callable):
+            what = 'function'
+        else:
+            what = 'object'
+
+    if what == 'class':
+        return ClassDoc(obj, func_doc=FunctionDoc, doc=doc, config=config)
+    elif what in ('function', 'method'):
+        return FunctionDoc(obj, doc=doc, config=config)
+    else:
+        if doc is None:
+            doc = pydoc.getdoc(obj)
+        return ObjDoc(obj, doc, config=config)

numpydoc/docscrape_sphinx.py

@@ -10,7 +10,7 @@
 import sphinx
 from sphinx.jinja2glue import BuiltinTemplateLoader
 
-from .docscrape import NumpyDocString, FunctionDoc, ClassDoc
+from .docscrape import NumpyDocString, FunctionDoc, ClassDoc, ObjDoc
 from .xref import make_xref
 
 
@@ -229,14 +229,6 @@ def _str_param_list(self, name, fake_autosummary=False):
 
         return out
 
-    @property
-    def _obj(self):
-        if hasattr(self, '_cls'):
-            return self._cls
-        elif hasattr(self, '_f'):
-            return self._f
-        return None
-
     def _str_member_list(self, name):
         """
         Generate a member listing, autosummary:: table where possible,
@@ -411,13 +403,13 @@ def __init__(self, obj, doc=None, func_doc=None, config={}):
         ClassDoc.__init__(self, obj, doc=doc, func_doc=None, config=config)
 
 
-class SphinxObjDoc(SphinxDocString):
+class SphinxObjDoc(SphinxDocString, ObjDoc):
     def __init__(self, obj, doc=None, config={}):
-        self._f = obj
         self.load_config(config)
-        SphinxDocString.__init__(self, doc, config=config)
+        ObjDoc.__init__(self, obj, doc=doc, config=config)
 
 
+# TODO: refactor to use docscrape.get_doc_object
 def get_doc_object(obj, what=None, doc=None, config={}, builder=None):
     if what is None:
         if inspect.isclass(obj):

numpydoc/numpydoc.py

@@ -34,6 +34,7 @@
     raise RuntimeError("Sphinx 1.6.5 or newer is required")
 
 from .docscrape_sphinx import get_doc_object
+from .validate import validate, ERROR_MSGS
 from .xref import DEFAULT_LINKS
 from . import __version__
 
@@ -173,6 +174,23 @@ def mangle_docstrings(app, what, name, obj, options, lines):
             logger.error('[numpydoc] While processing docstring for %r', name)
             raise
 
+        # TODO: validation only applies to non-module docstrings?
+        if app.config.numpydoc_validate:
+            # TODO: Currently, all validation checks are run and only those
+            # selected via config are reported. It would be more efficient to
+            # only run the selected checks.
+            errors = validate(doc)["errors"]
+            if {err[0] for err in errors} & app.config.numpydoc_validation_checks:
+                msg = (
+                    f"[numpydoc] Validation warnings while processing "
+                    f"docstring for {name!r}:\n"
+                )
+                for err in errors:
+                    if err[0] in app.config.numpydoc_validation_checks:
+                        msg += f"  {err[0]}: {err[1]}\n"
+                logger.warning(msg)
+
+
     if (app.config.numpydoc_edit_link and hasattr(obj, '__name__') and
             obj.__name__):
         if hasattr(obj, '__module__'):
@@ -254,6 +272,8 @@ def setup(app, get_doc_object_=get_doc_object):
     app.add_config_value('numpydoc_xref_param_type', False, True)
     app.add_config_value('numpydoc_xref_aliases', dict(), True)
     app.add_config_value('numpydoc_xref_ignore', set(), True)
+    app.add_config_value('numpydoc_validate', False, True)
+    app.add_config_value('numpydoc_validation_checks', set(), True)
 
     # Extra mangling domains
     app.add_domain(NumpyPythonDomain)
@@ -278,6 +298,12 @@ def update_config(app, config=None):
             numpydoc_xref_aliases_complete[key] = value
     config.numpydoc_xref_aliases_complete = numpydoc_xref_aliases_complete
 
+    # Processing to determine whether numpydoc_validation_checks is treated
+    # as a blocklist or allowlist
+    if "all" in config.numpydoc_validation_checks:
+        block = deepcopy(config.numpydoc_validation_checks)
+        config.numpydoc_validation_checks = set(ERROR_MSGS.keys()) - block
+
 
 # ------------------------------------------------------------------------------
 # Docstring-mangling domains

numpydoc/tests/test_docscrape.py

@@ -1497,6 +1497,8 @@ class Config():
         def __init__(self, a, b):
             self.numpydoc_xref_aliases = a
             self.numpydoc_xref_aliases_complete = b
+            # numpydoc.update_config fails if this config option not present
+            self.numpydoc_validation_checks = set()
 
     xref_aliases_complete = deepcopy(DEFAULT_LINKS)
     for key in xref_aliases:

numpydoc/tests/test_numpydoc.py

@@ -1,8 +1,11 @@
 # -*- encoding:utf-8 -*-
+import pytest
+from io import StringIO
 from copy import deepcopy
 from numpydoc.numpydoc import mangle_docstrings, _clean_text_signature
 from numpydoc.xref import DEFAULT_LINKS
 from sphinx.ext.autodoc import ALL
+from sphinx.util import logging
 
 
 class MockConfig():
@@ -19,6 +22,7 @@ class MockConfig():
     numpydoc_edit_link = False
     numpydoc_citation_re = '[a-z0-9_.-]+'
     numpydoc_attributes_as_param_list = True
+    numpydoc_validate = False
 
 
 class MockBuilder():
@@ -30,9 +34,12 @@ class MockApp():
     builder = MockBuilder()
     translator = None
 
-
-app = MockApp()
-app.builder.app = app
+    def __init__(self):
+        self.builder.app = self
+        # Attrs required for logging
+        self.verbosity = 2
+        self._warncount = 0
+        self.warningiserror = False
 
 
 def test_mangle_docstrings():
@@ -92,6 +99,58 @@ def test_clean_text_signature():
     assert _clean_text_signature('func($self, *args)') == 'func(*args)'
 
 
+@pytest.fixture
+def f():
+    def _function_without_seealso_and_examples():
+        """
+        A function whose docstring has no examples or see also section.
+
+        Expect SA01 and EX01 errors if validation enabled.
+        """
+        pass
+    return _function_without_seealso_and_examples
+
+
+@pytest.mark.parametrize(
+    (
+        'numpydoc_validate',
+        'numpydoc_validation_checks',
+        'expected_warn',
+        'non_warnings',
+    ),
+    (
+        # Validation configured off - expect no warnings
+        (False, set(['SA01', 'EX01']), [], []),
+        # Validation on with expected warnings
+        (True, set(['SA01', 'EX01']), ('SA01', 'EX01'), []),
+        # Validation on with only one activated check
+        (True, set(['SA01']), ('SA01',), ('EX01',)),
+    ),
+)
+def test_mangle_docstring_validation_warnings(
+    f,
+    numpydoc_validate,
+    numpydoc_validation_checks,
+    expected_warn,
+    non_warnings,
+):
+    app = MockApp()
+    # Set up config for test
+    app.config.numpydoc_validate = numpydoc_validate
+    app.config.numpydoc_validation_checks = numpydoc_validation_checks
+    # Set up logging
+    status, warning = StringIO(), StringIO()
+    logging.setup(app, status, warning)
+    # Run mangle docstrings with the above configuration
+    mangle_docstrings(app, 'function', 'f', f, None, f.__doc__.split('\n'))
+    # Assert that all (and only) expected warnings are logged
+    warnings = warning.getvalue()
+    for w in expected_warn:
+        assert w in warnings
+    for w in non_warnings:
+        assert w not in warnings
+
+
 if __name__ == "__main__":
     import pytest
     pytest.main()

numpydoc/tests/test_validate.py

@@ -1312,12 +1312,12 @@ def test_bad_docstrings(self, capsys, klass, func, msgs):
             assert msg in " ".join(err[1] for err in result["errors"])
 
 
-class TestDocstringClass:
+class TestValidatorClass:
     @pytest.mark.parametrize("invalid_name", ["unknown_mod", "unknown_mod.MyClass"])
     def test_raises_for_invalid_module_name(self, invalid_name):
         msg = 'No module can be imported from "{}"'.format(invalid_name)
         with pytest.raises(ImportError, match=msg):
-            numpydoc.validate.Docstring(invalid_name)
+            numpydoc.validate.Validator._load_obj(invalid_name)
 
     @pytest.mark.parametrize(
         "invalid_name", ["datetime.BadClassName", "datetime.bad_method_name"]
@@ -1327,4 +1327,4 @@ def test_raises_for_invalid_attribute_name(self, invalid_name):
         obj_name, invalid_attr_name = name_components[-2], name_components[-1]
         msg = "'{}' has no attribute '{}'".format(obj_name, invalid_attr_name)
         with pytest.raises(AttributeError, match=msg):
-            numpydoc.validate.Docstring(invalid_name)
+            numpydoc.validate.Validator._load_obj(invalid_name)

numpydoc/tests/tinybuild/conf.py

@@ -22,3 +22,4 @@
 highlight_language = 'python3'
 numpydoc_class_members_toctree = False
 numpydoc_xref_param_type = True
+numpydoc_validate = True