ENH: Use flake8 to check for PEP8 violations in doctests

scripts/tests/test_validate_docstrings.py

@@ -225,8 +225,9 @@ def good_imports(self):
         Examples
         --------
         This example does not import pandas or import numpy.
-        >>> import time
         >>> import datetime
+        >>> datetime.MAXYEAR
+        9999
         """
         pass
 
@@ -596,6 +597,44 @@ def prefix_pandas(self):
         pass
 
 
+class BadExamples(object):
+
+    def unused_import(self):
+        """
+        Examples
+        --------
+        >>> import pandas as pdf
+        >>> df = pd.DataFrame(np.ones((3, 3)), columns=('a', 'b', 'c'))
+        """
+        pass
+
+    def missing_whitespace_around_arithmetic_operator(self):
+        """
+        Examples
+        --------
+        >>> 2+5
+        7
+        """
+        pass
+
+    def indentation_is_not_a_multiple_of_four(self):
+        """
+        Examples
+        --------
+        >>> if 2 + 5:
+        ...   pass
+        """
+        pass
+
+    def missing_whitespace_after_comma(self):
+        """
+        Examples
+        --------
+        >>> df = pd.DataFrame(np.ones((3,3)),columns=('a','b', 'c'))
+        """
+        pass
+
+
 class TestValidator(object):
 
     def _import_path(self, klass=None, func=None):
@@ -634,7 +673,7 @@ def test_good_class(self):
     @capture_stderr
     @pytest.mark.parametrize("func", [
         'plot', 'sample', 'random_letters', 'sample_values', 'head', 'head1',
-        'contains', 'mode'])
+        'contains', 'mode', 'good_imports'])
     def test_good_functions(self, func):
         errors = validate_one(self._import_path(
             klass='GoodDocStrings', func=func))['errors']
@@ -714,16 +753,25 @@ def test_bad_generic_functions(self, func):
                      marks=pytest.mark.xfail),
         # Examples tests
         ('BadGenericDocStrings', 'method',
-         ('numpy does not need to be imported in the examples,')),
+         ('numpy does not need to be imported in the examples',)),
         ('BadGenericDocStrings', 'method',
-         ('pandas does not need to be imported in the examples,')),
+         ('pandas does not need to be imported in the examples',)),
         # See Also tests
         ('BadSeeAlso', 'prefix_pandas',
          ('pandas.Series.rename in `See Also` section '
-          'does not need `pandas` prefix',))
+          'does not need `pandas` prefix',)),
+        # Examples tests
+        ('BadExamples', 'unused_import',
+         ('1 F401 \'pandas as pdf\' imported but unused',)),
+        ('BadExamples', 'indentation_is_not_a_multiple_of_four',
+         ('1 E111 indentation is not a multiple of four',)),
+        ('BadExamples', 'missing_whitespace_around_arithmetic_operator',
+         ('1 E226 missing whitespace around arithmetic operator',)),
+        ('BadExamples', 'missing_whitespace_after_comma',
+         ('3 E231 missing whitespace after \',\'',)),
     ])
     def test_bad_examples(self, capsys, klass, func, msgs):
-        result = validate_one(self._import_path(klass=klass, func=func))  # noqa:F821
+        result = validate_one(self._import_path(klass=klass, func=func))
         for msg in msgs:
             assert msg in ' '.join(result['errors'])
 

scripts/validate_docstrings.py

@@ -24,6 +24,10 @@
 import inspect
 import importlib
 import doctest
+import tempfile
+
+import flake8.main.application
+
 try:
     from io import StringIO
 except ImportError:
@@ -168,7 +172,7 @@ def _load_obj(name):
     @staticmethod
     def _to_original_callable(obj):
         """
-        Find the Python object that contains the source code ot the object.
+        Find the Python object that contains the source code of the object.
 
         This is useful to find the place in the source code (file and line
         number) where a docstring is defined. It does not currently work for
@@ -407,6 +411,26 @@ def examples_source_code(self):
         lines = doctest.DocTestParser().get_examples(self.raw_doc)
         return [line.source for line in lines]
 
+    def validate_pep8(self):
+        if not self.examples:
+            return
+
+        content = ''.join(('import numpy as np  # noqa: F401\n',
+                           'import pandas as pd  # noqa: F401\n',
+                           *self.examples_source_code))
+
+        application = flake8.main.application.Application()
+        application.initialize(["--quiet"])
+
+        with tempfile.NamedTemporaryFile(mode='w') as file:
+            file.write(content)
+            file.flush()
+            application.run_checks([file.name])
+
+        application.report()
+
+        yield from application.guide.stats.statistics_for('')
+
 
 def validate_one(func_name):
     """
@@ -495,6 +519,13 @@ def validate_one(func_name):
         for param_err in param_errs:
             errs.append('\t{}'.format(param_err))
 
+    pep8_errs = list(doc.validate_pep8())
+    if pep8_errs:
+        errs.append('Linting issues in doctests:')
+        for err in pep8_errs:
+            errs.append('\t{} {} {}'.format(err.count, err.error_code,
+                                            err.message))
+
     if doc.is_function_or_method:
         if not doc.returns and "return" in doc.method_source:
             errs.append('No Returns section found')