From e1dee07679a8a47fcdadfdbba66aff2c4d2ce475 Mon Sep 17 00:00:00 2001 From: Honfung Wong <21543236+onshek@users.noreply.github.com> Date: Thu, 15 Oct 2020 01:21:26 +0800 Subject: [PATCH 01/10] UPD: check for numpy.random-related imports --- ci/code_checks.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ci/code_checks.sh b/ci/code_checks.sh index 784d2c9a411ab..f5cf9b278936e 100755 --- a/ci/code_checks.sh +++ b/ci/code_checks.sh @@ -197,6 +197,11 @@ if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then invgrep -r -E --include '*.py' --exclude testing.py '(numpy|np)(\.testing|\.array_equal)' pandas/tests/ RET=$(($RET + $?)) ; echo $MSG "DONE" + # GH37053, Check for np.random. instead of `from numpy.random import` or `from numpy import random` + MSG='Check for numpy.random-related imports' ; echo $MSG + invgrep -R --include="*.py" -E "(from numpy.random import)|(from numpy import random)" pandas + RET=$(($RET + $?)) ; echo $MSG "DONE" + # Check for the following code in the extension array base tests: `tm.assert_frame_equal` and `tm.assert_series_equal` MSG='Check for invalid EA testing' ; echo $MSG invgrep -r -E --include '*.py' --exclude base.py 'tm.assert_(series|frame)_equal' pandas/tests/extension/base From a07c28737e41b7f4b45d2b210c358463c41fbc41 Mon Sep 17 00:00:00 2001 From: Honfung Wong <21543236+onshek@users.noreply.github.com> Date: Sun, 25 Oct 2020 11:46:03 +0800 Subject: [PATCH 02/10] UPD: code_checks.sh --- ci/code_checks.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/code_checks.sh b/ci/code_checks.sh index f5cf9b278936e..794f35a637d3e 100755 --- a/ci/code_checks.sh +++ b/ci/code_checks.sh @@ -199,7 +199,7 @@ if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then # GH37053, Check for np.random. instead of `from numpy.random import` or `from numpy import random` MSG='Check for numpy.random-related imports' ; echo $MSG - invgrep -R --include="*.py" -E "(from numpy.random import)|(from numpy import random)" pandas + invgrep -R --include="*.py" --exclude=_testing.py -E "(from numpy.random import)|(from numpy import random)" pandas RET=$(($RET + $?)) ; echo $MSG "DONE" # Check for the following code in the extension array base tests: `tm.assert_frame_equal` and `tm.assert_series_equal` From 3f6223df7fcd1c46fe43f03c8f911b5e53fb351a Mon Sep 17 00:00:00 2001 From: Honfung Wong <21543236+onshek@users.noreply.github.com> Date: Sat, 31 Oct 2020 15:13:34 +0800 Subject: [PATCH 03/10] Revert "UPD: code_checks.sh" This reverts commit a07c28737e41b7f4b45d2b210c358463c41fbc41. --- ci/code_checks.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/code_checks.sh b/ci/code_checks.sh index 794f35a637d3e..f5cf9b278936e 100755 --- a/ci/code_checks.sh +++ b/ci/code_checks.sh @@ -199,7 +199,7 @@ if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then # GH37053, Check for np.random. instead of `from numpy.random import` or `from numpy import random` MSG='Check for numpy.random-related imports' ; echo $MSG - invgrep -R --include="*.py" --exclude=_testing.py -E "(from numpy.random import)|(from numpy import random)" pandas + invgrep -R --include="*.py" -E "(from numpy.random import)|(from numpy import random)" pandas RET=$(($RET + $?)) ; echo $MSG "DONE" # Check for the following code in the extension array base tests: `tm.assert_frame_equal` and `tm.assert_series_equal` From 8d85be3dfec1b38bae7c9d6698440fa3419e411a Mon Sep 17 00:00:00 2001 From: Honfung Wong <21543236+onshek@users.noreply.github.com> Date: Sat, 31 Oct 2020 15:13:39 +0800 Subject: [PATCH 04/10] Revert "UPD: check for numpy.random-related imports" This reverts commit e1dee07679a8a47fcdadfdbba66aff2c4d2ce475. --- ci/code_checks.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/ci/code_checks.sh b/ci/code_checks.sh index f5cf9b278936e..784d2c9a411ab 100755 --- a/ci/code_checks.sh +++ b/ci/code_checks.sh @@ -197,11 +197,6 @@ if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then invgrep -r -E --include '*.py' --exclude testing.py '(numpy|np)(\.testing|\.array_equal)' pandas/tests/ RET=$(($RET + $?)) ; echo $MSG "DONE" - # GH37053, Check for np.random. instead of `from numpy.random import` or `from numpy import random` - MSG='Check for numpy.random-related imports' ; echo $MSG - invgrep -R --include="*.py" -E "(from numpy.random import)|(from numpy import random)" pandas - RET=$(($RET + $?)) ; echo $MSG "DONE" - # Check for the following code in the extension array base tests: `tm.assert_frame_equal` and `tm.assert_series_equal` MSG='Check for invalid EA testing' ; echo $MSG invgrep -r -E --include '*.py' --exclude base.py 'tm.assert_(series|frame)_equal' pandas/tests/extension/base From 048354f347c2ede4d303a09424698ebd95c790f4 Mon Sep 17 00:00:00 2001 From: Honfung Wong <21543236+onshek@users.noreply.github.com> Date: Sat, 31 Oct 2020 16:35:55 +0800 Subject: [PATCH 05/10] UPD: with master --- .pre-commit-config.yaml | 70 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 65 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1c6d36133f067..d9e48ab9b6189 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,8 +15,7 @@ repos: - id: flake8 name: flake8 (cython template) files: \.pxi\.in$ - types: - - file + types: [text] args: [--append-config=flake8/cython-template.cfg] - repo: https://github.com/PyCQA/isort rev: 5.6.3 @@ -32,16 +31,20 @@ repos: - id: pyupgrade args: [--py37-plus] - repo: https://github.com/pre-commit/pygrep-hooks - rev: v1.6.0 + rev: v1.7.0 hooks: - id: rst-backticks + - id: rst-directive-colons + types: [text] + - id: rst-inline-touching-normal + types: [text] - repo: local hooks: - id: pip_to_conda name: Generate pip dependency from conda description: This hook checks if the conda environment.yml and requirements-dev.txt are equal language: python - entry: python -m scripts.generate_pip_deps_from_conda + entry: python scripts/generate_pip_deps_from_conda.py files: ^(environment.yml|requirements-dev.txt)$ pass_filenames: false additional_dependencies: [pyyaml] @@ -53,6 +56,61 @@ repos: types: [rst] args: [--filename=*.rst] additional_dependencies: [flake8-rst==0.7.0, flake8==3.7.9] + - id: non-standard-imports + name: Check for non-standard imports + language: pygrep + entry: | + (?x) + # Check for imports from pandas.core.common instead of `import pandas.core.common as com` + from\ pandas\.core\.common\ import| + from\ pandas\.core\ import\ common| + + # Check for imports from collections.abc instead of `from collections import abc` + from\ collections\.abc\ import| + + from\ numpy\ import\ nan + types: [python] + - id: non-standard-imports-in-tests + name: Check for non-standard imports in test suite + language: pygrep + entry: | + (?x) + # Check for imports from pandas._testing instead of `import pandas._testing as tm` + from\ pandas\._testing\ import| + from\ pandas\ import\ _testing\ as\ tm| + + # No direct imports from conftest + conftest\ import| + import\ conftest + types: [python] + files: ^pandas/tests/ + - id: incorrect-code-directives + name: Check for incorrect code block or IPython directives + language: pygrep + entry: (\.\. code-block ::|\.\. ipython ::) + files: \.(py|pyx|rst)$ + - id: unwanted-patterns-strings-to-concatenate + name: Check for use of not concatenated strings + language: python + entry: python scripts/validate_unwanted_patterns.py --validation-type="strings_to_concatenate" + files: \.(py|pyx|pxd|pxi)$ + - id: unwanted-patterns-strings-with-wrong-placed-whitespace + name: Check for strings with wrong placed spaces + language: python + entry: python scripts/validate_unwanted_patterns.py --validation-type="strings_with_wrong_placed_whitespace" + files: \.(py|pyx|pxd|pxi)$ + - id: unwanted-patterns-private-import-across-module + name: Check for import of private attributes across modules + language: python + entry: python scripts/validate_unwanted_patterns.py --validation-type="private_import_across_module" + types: [python] + exclude: ^(asv_bench|pandas/tests|doc)/ + - id: unwanted-patterns-private-function-across-module + name: Check for use of private functions across modules + language: python + entry: python scripts/validate_unwanted_patterns.py --validation-type="private_function_across_module" + types: [python] + exclude: ^(asv_bench|pandas/tests|doc)/ - repo: https://github.com/asottile/yesqa rev: v1.2.2 hooks: @@ -61,4 +119,6 @@ repos: rev: v3.2.0 hooks: - id: end-of-file-fixer - exclude: '.html$|^LICENSES/|.csv$|.txt$|.svg$|.py$' + exclude: ^LICENSES/|\.(html|csv|txt|svg|py)$ + - id: trailing-whitespace + exclude: \.(html|svg)$ From 88438b966f15e9a8e001661b4350afff3ca34f09 Mon Sep 17 00:00:00 2001 From: Honfung Wong <21543236+onshek@users.noreply.github.com> Date: Sat, 31 Oct 2020 16:38:43 +0800 Subject: [PATCH 06/10] Revert "UPD: with master" This reverts commit 048354f347c2ede4d303a09424698ebd95c790f4. --- .pre-commit-config.yaml | 70 +++-------------------------------------- 1 file changed, 5 insertions(+), 65 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d9e48ab9b6189..1c6d36133f067 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,7 +15,8 @@ repos: - id: flake8 name: flake8 (cython template) files: \.pxi\.in$ - types: [text] + types: + - file args: [--append-config=flake8/cython-template.cfg] - repo: https://github.com/PyCQA/isort rev: 5.6.3 @@ -31,20 +32,16 @@ repos: - id: pyupgrade args: [--py37-plus] - repo: https://github.com/pre-commit/pygrep-hooks - rev: v1.7.0 + rev: v1.6.0 hooks: - id: rst-backticks - - id: rst-directive-colons - types: [text] - - id: rst-inline-touching-normal - types: [text] - repo: local hooks: - id: pip_to_conda name: Generate pip dependency from conda description: This hook checks if the conda environment.yml and requirements-dev.txt are equal language: python - entry: python scripts/generate_pip_deps_from_conda.py + entry: python -m scripts.generate_pip_deps_from_conda files: ^(environment.yml|requirements-dev.txt)$ pass_filenames: false additional_dependencies: [pyyaml] @@ -56,61 +53,6 @@ repos: types: [rst] args: [--filename=*.rst] additional_dependencies: [flake8-rst==0.7.0, flake8==3.7.9] - - id: non-standard-imports - name: Check for non-standard imports - language: pygrep - entry: | - (?x) - # Check for imports from pandas.core.common instead of `import pandas.core.common as com` - from\ pandas\.core\.common\ import| - from\ pandas\.core\ import\ common| - - # Check for imports from collections.abc instead of `from collections import abc` - from\ collections\.abc\ import| - - from\ numpy\ import\ nan - types: [python] - - id: non-standard-imports-in-tests - name: Check for non-standard imports in test suite - language: pygrep - entry: | - (?x) - # Check for imports from pandas._testing instead of `import pandas._testing as tm` - from\ pandas\._testing\ import| - from\ pandas\ import\ _testing\ as\ tm| - - # No direct imports from conftest - conftest\ import| - import\ conftest - types: [python] - files: ^pandas/tests/ - - id: incorrect-code-directives - name: Check for incorrect code block or IPython directives - language: pygrep - entry: (\.\. code-block ::|\.\. ipython ::) - files: \.(py|pyx|rst)$ - - id: unwanted-patterns-strings-to-concatenate - name: Check for use of not concatenated strings - language: python - entry: python scripts/validate_unwanted_patterns.py --validation-type="strings_to_concatenate" - files: \.(py|pyx|pxd|pxi)$ - - id: unwanted-patterns-strings-with-wrong-placed-whitespace - name: Check for strings with wrong placed spaces - language: python - entry: python scripts/validate_unwanted_patterns.py --validation-type="strings_with_wrong_placed_whitespace" - files: \.(py|pyx|pxd|pxi)$ - - id: unwanted-patterns-private-import-across-module - name: Check for import of private attributes across modules - language: python - entry: python scripts/validate_unwanted_patterns.py --validation-type="private_import_across_module" - types: [python] - exclude: ^(asv_bench|pandas/tests|doc)/ - - id: unwanted-patterns-private-function-across-module - name: Check for use of private functions across modules - language: python - entry: python scripts/validate_unwanted_patterns.py --validation-type="private_function_across_module" - types: [python] - exclude: ^(asv_bench|pandas/tests|doc)/ - repo: https://github.com/asottile/yesqa rev: v1.2.2 hooks: @@ -119,6 +61,4 @@ repos: rev: v3.2.0 hooks: - id: end-of-file-fixer - exclude: ^LICENSES/|\.(html|csv|txt|svg|py)$ - - id: trailing-whitespace - exclude: \.(html|svg)$ + exclude: '.html$|^LICENSES/|.csv$|.txt$|.svg$|.py$' From 16d7df8e55ae9d31e5bf597a7a1c6438fc8c704f Mon Sep 17 00:00:00 2001 From: Honfung Wong <21543236+onshek@users.noreply.github.com> Date: Sat, 31 Oct 2020 16:44:17 +0800 Subject: [PATCH 07/10] ADD: check for imports from np.random. --- .pre-commit-config.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1c6d36133f067..cc68e27d3f929 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -53,6 +53,16 @@ repos: types: [rst] args: [--filename=*.rst] additional_dependencies: [flake8-rst==0.7.0, flake8==3.7.9] + + - id: non-standard-imports + name: Check for non-standard imports + language: pygrep + entry: | + (?x) + # Check for imports from np.random. instead of `from numpy import random` or `from numpy.random import ` + from\ numpy\ import\ random| + from\ numpy.random\ import + - repo: https://github.com/asottile/yesqa rev: v1.2.2 hooks: From c551e16e2e27a4d3e3d2447b20059a7060120fb8 Mon Sep 17 00:00:00 2001 From: Honfung Wong <21543236+onshek@users.noreply.github.com> Date: Sat, 31 Oct 2020 16:51:15 +0800 Subject: [PATCH 08/10] UPD: delete several blank lines --- .pre-commit-config.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9d6afa8ff0e57..70b2f9a140ecd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -56,13 +56,11 @@ repos: types: [rst] args: [--filename=*.rst] additional_dependencies: [flake8-rst==0.7.0, flake8==3.7.9] - - id: non-standard-imports name: Check for non-standard imports language: pygrep entry: | (?x) - # Check for imports from pandas.core.common instead of `import pandas.core.common as com` from\ pandas\.core\.common\ import| from\ pandas\.core\ import\ common| @@ -115,7 +113,6 @@ repos: entry: python scripts/validate_unwanted_patterns.py --validation-type="private_function_across_module" types: [python] exclude: ^(asv_bench|pandas/tests|doc)/ - - repo: https://github.com/asottile/yesqa rev: v1.2.2 hooks: From fd1acd6742e6064a2e7757e35efd6b08dfb36724 Mon Sep 17 00:00:00 2001 From: Honfung Wong <21543236+onshek@users.noreply.github.com> Date: Sat, 31 Oct 2020 17:17:47 +0800 Subject: [PATCH 09/10] ADD: exclude pandas/_testing.py --- .pre-commit-config.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 70b2f9a140ecd..6fac308679b10 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -66,8 +66,14 @@ repos: from\ pandas\.core\ import\ common| # Check for imports from collections.abc instead of `from collections import abc` - from\ collections\.abc\ import| + from\ collections\.abc\ import + - id: non-standard-numpy.random-related-imports + name: Check for non-standard numpy.random-related imports + language: pygrep + exclude: pandas/_testing.py + entry: | + (?x) # Check for imports from np.random. instead of `from numpy import random` or `from numpy.random import ` from\ numpy\ import\ random| from\ numpy.random\ import From 1e52ea5bcfe0a5f7d2f74ef243705a60138737fd Mon Sep 17 00:00:00 2001 From: Honfung Wong <21543236+onshek@users.noreply.github.com> Date: Sat, 31 Oct 2020 17:27:01 +0800 Subject: [PATCH 10/10] UPD: name of the new check --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6fac308679b10..1065ccff32632 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -69,12 +69,12 @@ repos: from\ collections\.abc\ import - id: non-standard-numpy.random-related-imports - name: Check for non-standard numpy.random-related imports + name: Check for non-standard numpy.random-related imports excluding pandas/_testing.py language: pygrep exclude: pandas/_testing.py entry: | (?x) - # Check for imports from np.random. instead of `from numpy import random` or `from numpy.random import ` + # Check for imports from np.random. instead of `from numpy import random` or `from numpy.random import ` from\ numpy\ import\ random| from\ numpy.random\ import types: [python]