From 5b482fed8b04b6296d39893dab66e4fe65529978 Mon Sep 17 00:00:00 2001 From: Ketu Patel Date: Fri, 28 Apr 2023 10:42:20 +0530 Subject: [PATCH 1/3] Changes Confirmed --- .pre-commit-config.yaml => b/.pre-commit-config.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) rename .pre-commit-config.yaml => b/.pre-commit-config.yaml (98%) diff --git a/.pre-commit-config.yaml b/b/.pre-commit-config.yaml similarity index 98% rename from .pre-commit-config.yaml rename to b/.pre-commit-config.yaml index 43b3699907325..fae32b7b01485 100644 --- a/.pre-commit-config.yaml +++ b/b/.pre-commit-config.yaml @@ -83,9 +83,6 @@ repos: hooks: - id: pylint stages: [manual] -- repo: https://github.com/pycqa/pylint - rev: v2.16.2 - hooks: - id: pylint alias: redefined-outer-name name: Redefining name from outer scope @@ -99,6 +96,11 @@ repos: |^pandas/conftest\.py # keep excluded args: [--disable=all, --enable=redefined-outer-name] stages: [manual] + - id: pylint + alias: unspecified-encoding + name: Using open without explicitly specifying an encoding + args: [--disable=all, --enable=unspecified-encoding] + stages: [manual] - repo: https://github.com/PyCQA/isort rev: 5.12.0 hooks: From c20ac6d72093019999c42c56eb1650adc90f750e Mon Sep 17 00:00:00 2001 From: Ketu Patel Date: Sat, 29 Apr 2023 14:30:48 +0530 Subject: [PATCH 2/3] Encoding Completed --- .pre-commit-config.yaml | 432 +++++++++++++++++++ scripts/check_test_naming.py | 2 +- scripts/validate_rst_title_capitalization.py | 2 +- 3 files changed, 434 insertions(+), 2 deletions(-) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000000..fae32b7b01485 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,432 @@ +minimum_pre_commit_version: 2.15.0 +exclude: ^LICENSES/|\.(html|csv|svg)$ +# reserve "manual" for relatively slow hooks which we still want to run in CI +default_stages: [ + commit, + merge-commit, + push, + prepare-commit-msg, + commit-msg, + post-checkout, + post-commit, + post-merge, + post-rewrite +] +ci: + autofix_prs: false +repos: +- repo: local + hooks: + # NOTE: we make `black` a local hook because if it's installed from + # PyPI (rather than from source) then it'll run twice as fast thanks to mypyc + - id: black + name: black + description: "Black: The uncompromising Python code formatter" + entry: black + language: python + require_serial: true + types_or: [python, pyi] + additional_dependencies: [black==23.1.0] +- repo: https://github.com/charliermarsh/ruff-pre-commit + rev: v0.0.259 + hooks: + - id: ruff + args: [--exit-non-zero-on-fix] +- repo: https://github.com/jendrikseipp/vulture + rev: 'v2.7' + hooks: + - id: vulture + entry: python scripts/run_vulture.py + pass_filenames: true + require_serial: false +- repo: https://github.com/codespell-project/codespell + rev: v2.2.2 + hooks: + - id: codespell + types_or: [python, rst, markdown, cython, c] + additional_dependencies: [tomli] +- repo: https://github.com/MarcoGorelli/cython-lint + rev: v0.12.5 + hooks: + - id: cython-lint + - id: double-quote-cython-strings +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: debug-statements + - id: end-of-file-fixer + exclude: \.txt$ + stages: [commit, merge-commit, push, prepare-commit-msg, commit-msg, + post-checkout, post-commit, post-merge, post-rewrite] + - id: trailing-whitespace + stages: [commit, merge-commit, push, prepare-commit-msg, commit-msg, + post-checkout, post-commit, post-merge, post-rewrite] +- repo: https://github.com/cpplint/cpplint + rev: 1.6.1 + hooks: + - id: cpplint + # We don't lint all C files because we don't want to lint any that are built + # from Cython files nor do we want to lint C files that we didn't modify for + # this particular codebase (e.g. src/headers, src/klib). However, + # we can lint all header files since they aren't "generated" like C files are. + exclude: ^pandas/_libs/src/(klib|headers)/ + args: [ + --quiet, + '--extensions=c,h', + '--headers=h', + --recursive, + --linelength=88, + '--filter=-readability/casting,-runtime/int,-build/include_subdir,-readability/fn_size' + ] +- repo: https://github.com/pycqa/pylint + rev: v2.16.2 + hooks: + - id: pylint + stages: [manual] + - id: pylint + alias: redefined-outer-name + name: Redefining name from outer scope + files: ^pandas/ + exclude: | + (?x) + ^pandas/tests # keep excluded + |/_testing/ # keep excluded + |^pandas/util/_test_decorators\.py # keep excluded + |^pandas/_version\.py # keep excluded + |^pandas/conftest\.py # keep excluded + args: [--disable=all, --enable=redefined-outer-name] + stages: [manual] + - id: pylint + alias: unspecified-encoding + name: Using open without explicitly specifying an encoding + args: [--disable=all, --enable=unspecified-encoding] + stages: [manual] +- repo: https://github.com/PyCQA/isort + rev: 5.12.0 + hooks: + - id: isort +- repo: https://github.com/asottile/pyupgrade + rev: v3.3.1 + hooks: + - id: pyupgrade + args: [--py38-plus] +- repo: https://github.com/pre-commit/pygrep-hooks + rev: v1.10.0 + hooks: + - id: rst-backticks + - id: rst-directive-colons + types: [text] # overwrite types: [rst] + types_or: [python, rst] + - id: rst-inline-touching-normal + types: [text] # overwrite types: [rst] + types_or: [python, rst] +- repo: https://github.com/sphinx-contrib/sphinx-lint + rev: v0.6.7 + hooks: + - id: sphinx-lint +- repo: local + hooks: + - id: pyright + # note: assumes python env is setup and activated + name: pyright + entry: pyright + language: node + pass_filenames: false + types: [python] + stages: [manual] + additional_dependencies: &pyright_dependencies + - pyright@1.1.292 + - id: pyright_reportGeneralTypeIssues + # note: assumes python env is setup and activated + name: pyright reportGeneralTypeIssues + entry: pyright --skipunannotated -p pyright_reportGeneralTypeIssues.json --level warning + language: node + pass_filenames: false + types: [python] + stages: [manual] + additional_dependencies: *pyright_dependencies + - id: mypy + # note: assumes python env is setup and activated + name: mypy + entry: mypy + language: system + pass_filenames: false + types: [python] + stages: [manual] + - id: stubtest + # note: assumes python env is setup and activated + # note: requires pandas dev to be installed + name: mypy (stubtest) + entry: python + language: system + pass_filenames: false + types: [pyi] + args: [scripts/run_stubtest.py] + stages: [manual] + - id: inconsistent-namespace-usage + name: 'Check for inconsistent use of pandas namespace' + entry: python scripts/check_for_inconsistent_pandas_namespace.py + exclude: ^pandas/core/interchange/ + language: python + types: [python] + - id: no-os-remove + name: Check code for instances of os.remove + entry: os\.remove + language: pygrep + types: [python] + files: ^pandas/tests/ + exclude: | + (?x)^ + pandas/tests/io/pytables/test_store\.py$ + - id: unwanted-patterns + name: Unwanted patterns + language: pygrep + entry: | + (?x) + # outdated annotation syntax, missing error codes + \#\ type:\ (?!ignore) + |\#\ type:\s?ignore(?!\[) + + # foo._class__ instead of type(foo) + |\.__class__ + + # np.bool/np.object instead of np.bool_/np.object_ + |np\.bool[^_8`] + |np\.object[^_8`] + + # imports from collections.abc instead of `from collections import abc` + |from\ collections\.abc\ import + + # Numpy + |from\ numpy\ import\ random + |from\ numpy\.random\ import + + # Incorrect code-block / IPython directives + |\.\.\ code-block\ :: + |\.\.\ ipython\ :: + # directive should not have a space before :: + |\.\.\ \w+\ :: + + # Check for deprecated messages without sphinx directive + |(DEPRECATED|DEPRECATE|Deprecated)(:|,|\.) + + # {foo!r} instead of {repr(foo)} + |!r} + + # builtin filter function + |(?obj`, not ` obj` + language: pygrep + entry: '[a-zA-Z0-9*]> ' + files: (\.pyx|\.pxi.in)$ + - id: incorrect-backticks + name: Check for backticks incorrectly rendering because of missing spaces + language: pygrep + entry: '[a-zA-Z0-9]\`\`?[a-zA-Z0-9]' + types: [rst] + files: ^doc/source/ + - id: seed-check-asv + name: Check for unnecessary random seeds in asv benchmarks + language: pygrep + entry: 'np\.random\.seed' + files: ^asv_bench/benchmarks + exclude: ^asv_bench/benchmarks/pandas_vb_common\.py + - id: np-testing-array-equal + name: Check for usage of numpy testing or array_equal + language: pygrep + entry: '(numpy|np)(\.testing|\.array_equal)' + files: ^pandas/tests/ + types: [python] + - id: invalid-ea-testing + name: Check for invalid EA testing + language: pygrep + entry: 'tm\.assert_(series|frame)_equal' + files: ^pandas/tests/extension/base + types: [python] + exclude: ^pandas/tests/extension/base/base\.py + - id: unwanted-patterns-in-tests + name: Unwanted patterns in tests + language: pygrep + entry: | + (?x) + # pytest.xfail instead of pytest.mark.xfail + pytest\.xfail + + # imports from pandas._testing instead of `import pandas._testing as tm` + |from\ pandas\._testing\ import + |from\ pandas\ import\ _testing\ as\ tm + + # No direct imports from conftest + |conftest\ import + |import\ conftest + + # pandas.testing instead of tm + |pd\.testing\. + + # pd.api.types instead of from pandas.api.types import ... + |(pd|pandas)\.api\.types\. + + # np.testing, np.array_equal + |(numpy|np)(\.testing|\.array_equal) + + # unittest.mock (use pytest builtin monkeypatch fixture instead) + |(unittest(\.| import )mock|mock\.Mock\(\)|mock\.patch) + + # pytest raises without context + |\s\ pytest.raises + + # pytest.warns (use tm.assert_produces_warning instead) + |pytest\.warns + files: ^pandas/tests/ + types_or: [python, cython, rst] + - id: unwanted-patterns-in-ea-tests + name: Unwanted patterns in EA tests + language: pygrep + entry: | + (?x) + tm.assert_(series|frame)_equal + files: ^pandas/tests/extension/base/ + exclude: ^pandas/tests/extension/base/base\.py$ + types_or: [python, cython, rst] + - id: unwanted-patterns-in-cython + name: Unwanted patterns in Cython code + language: pygrep + entry: | + (?x) + # `obj` as opposed to ` obj` + [a-zA-Z0-9*]>[ ] + types: [cython] + - id: pip-to-conda + name: Generate pip dependency from conda + language: python + entry: python scripts/generate_pip_deps_from_conda.py + files: ^(environment.yml|requirements-dev.txt)$ + pass_filenames: false + additional_dependencies: [tomli, pyyaml] + - id: title-capitalization + name: Validate correct capitalization among titles in documentation + entry: python scripts/validate_rst_title_capitalization.py + language: python + types: [rst] + files: ^doc/source/(development|reference)/ + - id: unwanted-patterns-bare-pytest-raises + name: Check for use of bare pytest raises + language: python + entry: python scripts/validate_unwanted_patterns.py --validation-type="bare_pytest_raises" + types: [python] + files: ^pandas/tests/ + exclude: ^pandas/tests/extension/ + - id: unwanted-patterns-private-function-across-module + name: Check for use of private functions across modules + language: python + entry: python scripts/validate_unwanted_patterns.py --validation-type="private_function_across_module" + types: [python] + exclude: ^(asv_bench|pandas/tests|doc)/ + - id: unwanted-patterns-private-import-across-module + name: Check for import of private attributes across modules + language: python + entry: python scripts/validate_unwanted_patterns.py --validation-type="private_import_across_module" + types: [python] + exclude: | + (?x) + ^(asv_bench|pandas/tests|doc)/ + |scripts/validate_min_versions_in_sync\.py$ + - id: unwanted-patterns-strings-with-misplaced-whitespace + name: Check for strings with misplaced spaces + language: python + entry: python scripts/validate_unwanted_patterns.py --validation-type="strings_with_wrong_placed_whitespace" + types_or: [python, cython] + - id: use-pd_array-in-core + name: Import pandas.array as pd_array in core + language: python + entry: python scripts/use_pd_array_in_core.py + files: ^pandas/core/ + exclude: ^pandas/core/api\.py$ + types: [python] + - id: use-io-common-urlopen + name: Use pandas.io.common.urlopen instead of urllib.request.urlopen + language: python + entry: python scripts/use_io_common_urlopen.py + files: ^pandas/ + exclude: ^pandas/tests/ + types: [python] + - id: no-bool-in-core-generic + name: Use bool_t instead of bool in pandas/core/generic.py + entry: python scripts/no_bool_in_generic.py + language: python + files: ^pandas/core/generic\.py$ + - id: no-return-exception + name: Use raise instead of return for exceptions + language: pygrep + entry: 'return [A-Za-z]+(Error|Exit|Interrupt|Exception|Iteration)' + files: ^pandas/ + types: [python] + exclude: ^pandas/tests/ + - id: pandas-errors-documented + name: Ensure pandas errors are documented in doc/source/reference/testing.rst + entry: python scripts/pandas_errors_documented.py + language: python + files: ^pandas/errors/__init__.py$ + - id: pg8000-not-installed-CI + name: Check for pg8000 not installed on CI for test_pg8000_sqlalchemy_passthrough_error + language: pygrep + entry: 'pg8000' + files: ^ci/deps + types: [yaml] + - id: validate-min-versions-in-sync + name: Check minimum version of dependencies are aligned + entry: python -m scripts.validate_min_versions_in_sync + language: python + files: ^(ci/deps/actions-.*-minimum_versions\.yaml|pandas/compat/_optional\.py)$ + additional_dependencies: [tomli, pyyaml] + pass_filenames: false + - id: validate-errors-locations + name: Validate errors locations + description: Validate errors are in appropriate locations. + entry: python scripts/validate_exception_location.py + language: python + files: ^pandas/ + exclude: ^(pandas/_libs/|pandas/tests/|pandas/errors/__init__.py$|pandas/_version.py) + types: [python] + - id: future-annotations + name: import annotations from __future__ + entry: 'from __future__ import annotations' + language: pygrep + args: [--negate] + files: ^pandas/ + types: [python] + exclude: | + (?x) + /(__init__\.py)|(api\.py)|(_version\.py)|(testing\.py)|(conftest\.py)$ + |/tests/ + |/_testing/ + - id: autotyping + name: autotyping + entry: python -m scripts.run_autotyping + types_or: [python, pyi] + files: ^pandas + exclude: ^(pandas/tests|pandas/_version.py|pandas/io/clipboard) + language: python + stages: [manual] + additional_dependencies: + - autotyping==23.3.0 + - libcst==0.4.9 + - id: check-test-naming + name: check that test names start with 'test' + entry: python -m scripts.check_test_naming + types: [python] + files: ^pandas/tests + language: python + - id: sort-whatsnew-items + name: sort whatsnew entries alphabetically + entry: python -m scripts.sort_whatsnew_note + types: [rst] + language: python + files: ^doc/source/whatsnew/v + exclude: ^doc/source/whatsnew/v(0|1|2\.0\.0) diff --git a/scripts/check_test_naming.py b/scripts/check_test_naming.py index 33890feb8692d..158cf46f264c2 100644 --- a/scripts/check_test_naming.py +++ b/scripts/check_test_naming.py @@ -118,7 +118,7 @@ def main(content: str, file: str) -> int: assert isinstance(_node, ast.FunctionDef) # help mypy should_continue = False for _file in (Path("pandas") / "tests").rglob("*.py"): - with open(os.path.join(_file)) as fd: + with open(os.path.join(_file), encoding="utf-8") as fd: _content = fd.read() if f"self.{_node.name}" in _content: should_continue = True diff --git a/scripts/validate_rst_title_capitalization.py b/scripts/validate_rst_title_capitalization.py index 4446ed62f6b8a..0f4c11eb30b07 100755 --- a/scripts/validate_rst_title_capitalization.py +++ b/scripts/validate_rst_title_capitalization.py @@ -226,7 +226,7 @@ def find_titles(rst_file: str) -> Iterable[tuple[str, int]]: The corresponding line number of the heading. """ - with open(rst_file) as fd: + with open(rst_file, encoding="utf-8") as fd: previous_line = "" for i, line in enumerate(fd): line_no_last_elem = line[:-1] From 682fdf836dbb96b55c2ba4f90558eef608c6700b Mon Sep 17 00:00:00 2001 From: Ketu Patel Date: Sat, 29 Apr 2023 15:25:59 +0530 Subject: [PATCH 3/3] Spaces Are Completed --- .pre-commit-config.yaml | 10 +++++----- b/.pre-commit-config.yaml | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fae32b7b01485..34cd91940f014 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -96,11 +96,11 @@ repos: |^pandas/conftest\.py # keep excluded args: [--disable=all, --enable=redefined-outer-name] stages: [manual] - - id: pylint - alias: unspecified-encoding - name: Using open without explicitly specifying an encoding - args: [--disable=all, --enable=unspecified-encoding] - stages: [manual] + - id: pylint + alias: unspecified-encoding + name: Using open without explicitly specifying an encoding + args: [--disable=all, --enable=unspecified-encoding] + stages: [manual] - repo: https://github.com/PyCQA/isort rev: 5.12.0 hooks: diff --git a/b/.pre-commit-config.yaml b/b/.pre-commit-config.yaml index fae32b7b01485..34cd91940f014 100644 --- a/b/.pre-commit-config.yaml +++ b/b/.pre-commit-config.yaml @@ -96,11 +96,11 @@ repos: |^pandas/conftest\.py # keep excluded args: [--disable=all, --enable=redefined-outer-name] stages: [manual] - - id: pylint - alias: unspecified-encoding - name: Using open without explicitly specifying an encoding - args: [--disable=all, --enable=unspecified-encoding] - stages: [manual] + - id: pylint + alias: unspecified-encoding + name: Using open without explicitly specifying an encoding + args: [--disable=all, --enable=unspecified-encoding] + stages: [manual] - repo: https://github.com/PyCQA/isort rev: 5.12.0 hooks: