Skip to content

CI move validate unwanted patterns over to pre-commit #37379

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Oct 26, 2020
Merged

CI move validate unwanted patterns over to pre-commit #37379

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e6ef655
move validate unwanted patterns over to pre-commit
MarcoGorelli Oct 24, 2020
d4f1bd3
better names
MarcoGorelli Oct 24, 2020
7d5620b
Merge remote-tracking branch 'upstream/master' into unwanted-patterns…
MarcoGorelli Oct 24, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions .pre-commit-config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,28 @@ repos:
language: pygrep
entry: (\.\. code-block ::|\.\. ipython ::)
files: \.(py|pyx|rst)$
- id: unwanted-patterns-strings-to-concatenate
name: Check for use of not concatenated strings
language: python
entry: ./scripts/validate_unwanted_patterns.py --validation-type="strings_to_concatenate"
files: \.(py|pyx|pxd|pxi)$
- id: unwanted-patterns-strings-with-wrong-placed-whitespace
name: Check for strings with wrong placed spaces
language: python
entry: ./scripts/validate_unwanted_patterns.py --validation-type="strings_with_wrong_placed_whitespace"
files: \.(py|pyx|pxd|pxi)$
- id: unwanted-patterns-private-import-across-module
name: Check for import of private attributes across modules
language: python
entry: ./scripts/validate_unwanted_patterns.py --validation-type="private_import_across_module"
types: [python]
exclude: ^(asv_bench|pandas/_vendored|pandas/tests|doc)/
- id: unwanted-patterns-private-function-across-module
name: Check for use of private functions across modules
language: python
entry: ./scripts/validate_unwanted_patterns.py --validation-type="private_function_across_module"
types: [python]
exclude: ^(asv_bench|pandas/_vendored|pandas/tests|doc)/
- repo: https://github.com/asottile/yesqa
rev: v1.2.2
hooks:
Expand Down
32 changes: 0 additions & 32 deletions ci/code_checks.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,38 +73,6 @@ if [[ -z "$CHECK" || "$CHECK" == "lint" ]]; then
cpplint --quiet --extensions=c,h --headers=h --recursive --filter=-readability/casting,-runtime/int,-build/include_subdir pandas/_libs/src/*.h pandas/_libs/src/parser pandas/_libs/ujson pandas/_libs/tslibs/src/datetime pandas/_libs/*.cpp
RET=$(($RET + $?)) ; echo $MSG "DONE"

MSG='Check for use of not concatenated strings' ; echo $MSG
if [[ "$GITHUB_ACTIONS" == "true" ]]; then
$BASE_DIR/scripts/validate_unwanted_patterns.py --validation-type="strings_to_concatenate" --format="##[error]{source_path}:{line_number}:{msg}" .
else
$BASE_DIR/scripts/validate_unwanted_patterns.py --validation-type="strings_to_concatenate" .
fi
RET=$(($RET + $?)) ; echo $MSG "DONE"

MSG='Check for strings with wrong placed spaces' ; echo $MSG
if [[ "$GITHUB_ACTIONS" == "true" ]]; then
$BASE_DIR/scripts/validate_unwanted_patterns.py --validation-type="strings_with_wrong_placed_whitespace" --format="##[error]{source_path}:{line_number}:{msg}" .
else
$BASE_DIR/scripts/validate_unwanted_patterns.py --validation-type="strings_with_wrong_placed_whitespace" .
fi
RET=$(($RET + $?)) ; echo $MSG "DONE"

MSG='Check for import of private attributes across modules' ; echo $MSG
if [[ "$GITHUB_ACTIONS" == "true" ]]; then
$BASE_DIR/scripts/validate_unwanted_patterns.py --validation-type="private_import_across_module" --included-file-extensions="py" --excluded-file-paths=pandas/tests,asv_bench/,pandas/_vendored --format="##[error]{source_path}:{line_number}:{msg}" pandas/
else
$BASE_DIR/scripts/validate_unwanted_patterns.py --validation-type="private_import_across_module" --included-file-extensions="py" --excluded-file-paths=pandas/tests,asv_bench/,pandas/_vendored pandas/
fi
RET=$(($RET + $?)) ; echo $MSG "DONE"

MSG='Check for use of private functions across modules' ; echo $MSG
if [[ "$GITHUB_ACTIONS" == "true" ]]; then
$BASE_DIR/scripts/validate_unwanted_patterns.py --validation-type="private_function_across_module" --included-file-extensions="py" --excluded-file-paths=pandas/tests,asv_bench/,pandas/_vendored,doc/ --format="##[error]{source_path}:{line_number}:{msg}" pandas/
else
$BASE_DIR/scripts/validate_unwanted_patterns.py --validation-type="private_function_across_module" --included-file-extensions="py" --excluded-file-paths=pandas/tests,asv_bench/,pandas/_vendored,doc/ pandas/
fi
RET=$(($RET + $?)) ; echo $MSG "DONE"

fi

### PATTERNS ###
Expand Down
54 changes: 4 additions & 50 deletions scripts/validate_unwanted_patterns.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,10 @@

import argparse
import ast
import os
import sys
import token
import tokenize
from typing import IO, Callable, FrozenSet, Iterable, List, Set, Tuple
from typing import IO, Callable, Iterable, List, Set, Tuple

PRIVATE_IMPORTS_TO_IGNORE: Set[str] = {
"_extension_array_shared_docs",
Expand Down Expand Up @@ -403,8 +402,6 @@ def main(
function: Callable[[IO[str]], Iterable[Tuple[int, str]]],
source_path: str,
output_format: str,
file_extensions_to_check: str,
excluded_file_paths: str,
) -> bool:
"""
Main entry point of the script.
Expand Down Expand Up @@ -432,19 +429,9 @@ def main(
ValueError
If the `source_path` is not pointing to existing file/directory.
"""
if not os.path.exists(source_path):
raise ValueError("Please enter a valid path, pointing to a file/directory.")

is_failed: bool = False
file_path: str = ""

FILE_EXTENSIONS_TO_CHECK: FrozenSet[str] = frozenset(
file_extensions_to_check.split(",")
)
PATHS_TO_IGNORE = frozenset(excluded_file_paths.split(","))

if os.path.isfile(source_path):
file_path = source_path
for file_path in source_path:
with open(file_path) as file_obj:
for line_number, msg in function(file_obj):
is_failed = True
Expand All @@ -454,25 +441,6 @@ def main(
)
)

for subdir, _, files in os.walk(source_path):
if any(path in subdir for path in PATHS_TO_IGNORE):
continue
for file_name in files:
if not any(
file_name.endswith(extension) for extension in FILE_EXTENSIONS_TO_CHECK
):
continue

file_path = os.path.join(subdir, file_name)
with open(file_path) as file_obj:
for line_number, msg in function(file_obj):
is_failed = True
print(
output_format.format(
source_path=file_path, line_number=line_number, msg=msg
)
)

return is_failed


Expand All @@ -487,9 +455,7 @@ def main(

parser = argparse.ArgumentParser(description="Unwanted patterns checker.")

parser.add_argument(
"path", nargs="?", default=".", help="Source path of file/directory to check."
)
parser.add_argument("paths", nargs="*", help="Source paths of files to check.")
parser.add_argument(
"--format",
"-f",
Expand All @@ -503,25 +469,13 @@ def main(
required=True,
help="Validation test case to check.",
)
parser.add_argument(
"--included-file-extensions",
default="py,pyx,pxd,pxi",
help="Comma separated file extensions to check.",
)
parser.add_argument(
"--excluded-file-paths",
default="asv_bench/env",
help="Comma separated file paths to exclude.",
)

args = parser.parse_args()

sys.exit(
main(
function=globals().get(args.validation_type), # type: ignore
source_path=args.path,
source_path=args.paths,
output_format=args.format,
file_extensions_to_check=args.included_file_extensions,
excluded_file_paths=args.excluded_file_paths,
)
)