Revert "STYLE use pandas-dev-flaker (pandas-dev#40906)"

This reverts commit 1f27ed0.

Author: MarcoGorelli

.pre-commit-config.yaml

@@ -70,12 +70,9 @@ repos:
     rev: 6.0.0
     hooks:
     -   id: flake8
-        # Need to patch os.remove rule in pandas-dev-flaker
-        exclude: ^ci/fix_wheels.py
         additional_dependencies: &flake8_dependencies
         - flake8==6.0.0
         - flake8-bugbear==22.7.1
-        - pandas-dev-flaker==0.5.0
 -   repo: https://github.com/pycqa/pylint
     rev: v2.15.6
     hooks:
@@ -184,6 +181,28 @@ repos:
         types: [rst]
         args: [--filename=*.rst]
         additional_dependencies: [flake8-rst==0.7.0, flake8==3.7.9]
+    -   id: frame-or-series-union
+        name: Check for use of Union[Series, DataFrame] instead of FrameOrSeriesUnion alias
+        entry: Union\[.*(Series,.*DataFrame|DataFrame,.*Series).*\]
+        language: pygrep
+        types: [python]
+        exclude: ^pandas/_typing\.py$
+    -   id: inconsistent-namespace-usage
+        name: 'Check for inconsistent use of pandas namespace'
+        entry: python scripts/check_for_inconsistent_pandas_namespace.py
+        language: python
+        types: [python]
+    -   id: no-os-remove
+        name: Check code for instances of os.remove
+        entry: os\.remove
+        language: pygrep
+        types: [python]
+        files: ^pandas/tests/
+        exclude: |
+            (?x)^
+            pandas/tests/io/excel/test_writers\.py
+            |pandas/tests/io/pytables/common\.py
+            |pandas/tests/io/pytables/test_store\.py$
     -   id: unwanted-patterns
         name: Unwanted patterns
         language: pygrep
@@ -193,6 +212,24 @@ repos:
             \#\ type:\ (?!ignore)
             |\#\ type:\s?ignore(?!\[)
 
+            # foo._class__ instead of type(foo)
+            |\.__class__
+
+            # np.bool/np.object instead of np.bool_/np.object_
+            |np\.bool[^_8]
+            |np\.object[^_8]
+
+            # imports from pandas.core.common instead of `import pandas.core.common as com`
+            |from\ pandas\.core\.common\ import
+            |from\ pandas\.core\ import\ common
+
+            # imports from collections.abc instead of `from collections import abc`
+            |from\ collections\.abc\ import
+
+            # Numpy
+            |from\ numpy\ import\ random
+            |from\ numpy\.random\ import
+
             # Incorrect code-block / IPython directives
             |\.\.\ code-block\ ::
             |\.\.\ ipython\ ::
@@ -202,6 +239,7 @@ repos:
             # Check for deprecated messages without sphinx directive
             |(DEPRECATED|DEPRECATE|Deprecated)(:|,|\.)
         types_or: [python, cython, rst]
+        exclude: ^doc/source/development/code_style\.rst  # contains examples of patterns to avoid
     -   id: cython-casting
         name: Check Cython casting is `<type>obj`, not `<type> obj`
         language: pygrep
@@ -232,6 +270,29 @@ repos:
         files: ^pandas/tests/extension/base
         types: [python]
         exclude: ^pandas/tests/extension/base/base\.py
+    -   id: unwanted-patterns-in-tests
+        name: Unwanted patterns in tests
+        language: pygrep
+        entry: |
+            (?x)
+            # pytest.xfail instead of pytest.mark.xfail
+            pytest\.xfail
+
+            # imports from pandas._testing instead of `import pandas._testing as tm`
+            |from\ pandas\._testing\ import
+            |from\ pandas\ import\ _testing\ as\ tm
+
+            # No direct imports from conftest
+            |conftest\ import
+            |import\ conftest
+
+            # pandas.testing instead of tm
+            |pd\.testing\.
+
+            # pd.api.types instead of from pandas.api.types import ...
+            |(pd|pandas)\.api\.types\.
+        files: ^pandas/tests/
+        types_or: [python, cython, rst]
     -   id: pip-to-conda
         name: Generate pip dependency from conda
         language: python
@@ -252,6 +313,35 @@ repos:
         language: python
         types: [rst]
         files: ^doc/source/(development|reference)/
+    -   id: unwanted-patterns-bare-pytest-raises
+        name: Check for use of bare pytest raises
+        language: python
+        entry: python scripts/validate_unwanted_patterns.py --validation-type="bare_pytest_raises"
+        types: [python]
+        files: ^pandas/tests/
+        exclude: ^pandas/tests/extension/
+    -   id: unwanted-patterns-private-function-across-module
+        name: Check for use of private functions across modules
+        language: python
+        entry: python scripts/validate_unwanted_patterns.py --validation-type="private_function_across_module"
+        types: [python]
+        exclude: ^(asv_bench|pandas/tests|doc)/
+    -   id: unwanted-patterns-private-import-across-module
+        name: Check for import of private attributes across modules
+        language: python
+        entry: python scripts/validate_unwanted_patterns.py --validation-type="private_import_across_module"
+        types: [python]
+        exclude: ^(asv_bench|pandas/tests|doc)/
+    -   id: unwanted-patterns-strings-to-concatenate
+        name: Check for use of not concatenated strings
+        language: python
+        entry: python scripts/validate_unwanted_patterns.py --validation-type="strings_to_concatenate"
+        types_or: [python, cython]
+    -   id: unwanted-patterns-strings-with-wrong-placed-whitespace
+        name: Check for strings with wrong placed spaces
+        language: python
+        entry: python scripts/validate_unwanted_patterns.py --validation-type="strings_with_wrong_placed_whitespace"
+        types_or: [python, cython]
     -   id: use-pd_array-in-core
         name: Import pandas.array as pd_array in core
         language: python

asv_bench/benchmarks/pandas_vb_common.py

@@ -70,7 +70,7 @@ class BaseIO:
     def remove(self, f):
         """Remove created files"""
         try:
-            os.remove(f)  # noqa: PDF008
+            os.remove(f)
         except OSError:
             # On Windows, attempting to remove a file that is in use
             # causes an exception to be raised

ci/code_checks.sh

@@ -8,14 +8,16 @@
 #
 # Usage:
 #   $ ./ci/code_checks.sh               # run all checks
+#   $ ./ci/code_checks.sh lint          # run linting only
+#   $ ./ci/code_checks.sh patterns      # check for patterns that should not exist
 #   $ ./ci/code_checks.sh code          # checks on imported code
 #   $ ./ci/code_checks.sh doctests      # run doctests
 #   $ ./ci/code_checks.sh docstrings    # validate docstring errors
 #   $ ./ci/code_checks.sh single-docs   # check single-page docs build warning-free
 #   $ ./ci/code_checks.sh notebooks     # check execution of documentation notebooks
 
-[[ -z "$1" || "$1" == "code" || "$1" == "doctests" || "$1" == "docstrings" || "$1" == "single-docs" || "$1" == "notebooks" ]] || \
-    { echo "Unknown command $1. Usage: $0 [code|doctests|docstrings|single-docs|notebooks]"; exit 9999; }
+[[ -z "$1" || "$1" == "lint" || "$1" == "patterns" || "$1" == "code" || "$1" == "doctests" || "$1" == "docstrings" || "$1" == "single-docs" || "$1" == "notebooks" ]] || \
+    { echo "Unknown command $1. Usage: $0 [lint|patterns|code|doctests|docstrings|single-docs|notebooks]"; exit 9999; }
 
 BASE_DIR="$(dirname $0)/.."
 RET=0
@@ -38,6 +40,71 @@ if [[ "$GITHUB_ACTIONS" == "true" ]]; then
     INVGREP_PREPEND="##[error]"
 fi
 
+### LINTING ###
+if [[ -z "$CHECK" || "$CHECK" == "lint" ]]; then
+
+    # Check that cython casting is of the form `<type>obj` as opposed to `<type> obj`;
+    # it doesn't make a difference, but we want to be internally consistent.
+    # Note: this grep pattern is (intended to be) equivalent to the python
+    # regex r'(?<![ ->])> '
+    MSG='Linting .pyx code for spacing conventions in casting' ; echo $MSG
+    invgrep -r -E --include '*.pyx' --include '*.pxi.in' '[a-zA-Z0-9*]> ' pandas/_libs
+    RET=$(($RET + $?)) ; echo $MSG "DONE"
+
+    # readability/casting: Warnings about C casting instead of C++ casting
+    # runtime/int: Warnings about using C number types instead of C++ ones
+    # build/include_subdir: Warnings about prefacing included header files with directory
+
+fi
+
+### PATTERNS ###
+if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then
+
+    MSG='Check for use of exec' ; echo $MSG
+    invgrep -R --include="*.py*" -E "[^a-zA-Z0-9_]exec\(" pandas
+    RET=$(($RET + $?)) ; echo $MSG "DONE"
+
+    MSG='Check for pytest warns' ; echo $MSG
+    invgrep -r -E --include '*.py' 'pytest\.warns' pandas/tests/
+    RET=$(($RET + $?)) ; echo $MSG "DONE"
+
+    MSG='Check for pytest raises without context' ; echo $MSG
+    invgrep -r -E --include '*.py' "[[:space:]] pytest.raises" pandas/tests/
+    RET=$(($RET + $?)) ; echo $MSG "DONE"
+
+    MSG='Check for use of builtin filter function' ; echo $MSG
+    invgrep -R --include="*.py" -P '(?<!def)[\(\s]filter\(' pandas
+    RET=$(($RET + $?)) ; echo $MSG "DONE"
+
+    # Check for the following code in testing: `np.testing` and `np.array_equal`
+    MSG='Check for invalid testing' ; echo $MSG
+    invgrep -r -E --include '*.py' --exclude testing.py '(numpy|np)(\.testing|\.array_equal)' pandas/tests/
+    RET=$(($RET + $?)) ; echo $MSG "DONE"
+
+    # Check for the following code in the extension array base tests: `tm.assert_frame_equal` and `tm.assert_series_equal`
+    MSG='Check for invalid EA testing' ; echo $MSG
+    invgrep -r -E --include '*.py' --exclude base.py 'tm.assert_(series|frame)_equal' pandas/tests/extension/base
+    RET=$(($RET + $?)) ; echo $MSG "DONE"
+
+    MSG='Check for deprecated messages without sphinx directive' ; echo $MSG
+    invgrep -R --include="*.py" --include="*.pyx" -E "(DEPRECATED|DEPRECATE|Deprecated)(:|,|\.)" pandas
+    RET=$(($RET + $?)) ; echo $MSG "DONE"
+
+    MSG='Check for backticks incorrectly rendering because of missing spaces' ; echo $MSG
+    invgrep -R --include="*.rst" -E "[a-zA-Z0-9]\`\`?[a-zA-Z0-9]" doc/source/
+    RET=$(($RET + $?)) ; echo $MSG "DONE"
+
+    # Check for the following code in testing: `unittest.mock`, `mock.Mock()` or `mock.patch`
+    MSG='Check that unittest.mock is not used (pytest builtin monkeypatch fixture should be used instead)' ; echo $MSG
+    invgrep -r -E --include '*.py' '(unittest(\.| import )mock|mock\.Mock\(\)|mock\.patch)' pandas/tests/
+    RET=$(($RET + $?)) ; echo $MSG "DONE"
+
+    MSG='Check for use of {foo!r} instead of {repr(foo)}' ; echo $MSG
+    invgrep -R --include=*.{py,pyx} '!r}' pandas
+    RET=$(($RET + $?)) ; echo $MSG "DONE"
+    echo $MSG "DONE"
+fi
+
 ### CODE ###
 if [[ -z "$CHECK" || "$CHECK" == "code" ]]; then