From b9ca491a0f4db5628447f29d422464d47724c758 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torsten=20W=C3=B6rtwein?= Date: Fri, 20 May 2022 12:17:32 -0400 Subject: [PATCH 1/5] CI: mypy&pyright in manual pre-commit stage --- .github/workflows/code-checks.yml | 21 ++++++++++----------- .pre-commit-config.yaml | 12 +++++++++++- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/.github/workflows/code-checks.yml b/.github/workflows/code-checks.yml index 7d24b26f5538b..da7c77c34a133 100644 --- a/.github/workflows/code-checks.yml +++ b/.github/workflows/code-checks.yml @@ -67,15 +67,6 @@ jobs: environment-file: ${{ env.ENV_FILE }} use-only-tar-bz2: true - - name: Install node.js (for pyright) - uses: actions/setup-node@v3 - with: - node-version: "16" - - - name: Install pyright - # note: keep version in sync with .pre-commit-config.yaml - run: npm install -g pyright@1.1.247 - - name: Build Pandas id: build uses: ./.github/actions/build_pandas @@ -96,8 +87,16 @@ jobs: run: ci/code_checks.sh docstrings if: ${{ steps.build.outcome == 'success' }} - - name: Run typing validation - run: ci/code_checks.sh typing + - name: Run typing validation (mypy) + uses: pre-commit/action@v2.0.3 + with: + extra_args: --hook-stage manual --all-files mypy + if: ${{ steps.build.outcome == 'success' }} + + - name: Run typing validation (pyright) + uses: pre-commit/action@v2.0.3 + with: + extra_args: --hook-stage manual --all-files pyright if: ${{ steps.build.outcome == 'success' }} - name: Run docstring validation script tests diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9469a34c8aacd..fc92119ca4160 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -84,12 +84,22 @@ repos: - id: pyright name: pyright entry: pyright + # note: assumes python env is setup and activated language: node pass_filenames: false types: [python] stages: [manual] - # note: keep version in sync with .github/workflows/code-checks.yml additional_dependencies: ['pyright@1.1.247'] +- repo: local + hooks: + - id: mypy + name: mypy + entry: mypy + # note: assumes python env is setup and activated + language: system + pass_filenames: false + types: [python] + stages: [manual] - repo: local hooks: - id: flake8-rst From cc0c6f04a95b5320de0e33ba9dba2bc5f5d4b362 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torsten=20W=C3=B6rtwein?= Date: Fri, 20 May 2022 14:43:01 -0400 Subject: [PATCH 2/5] move mypy+pyright to pre-commit --- .github/workflows/code-checks.yml | 18 +++---------- .pre-commit-config.yaml | 44 ++++++++++++++++++++++--------- run_pyright.sh | 25 ++++++++++++++++++ 3 files changed, 61 insertions(+), 26 deletions(-) create mode 100755 run_pyright.sh diff --git a/.github/workflows/code-checks.yml b/.github/workflows/code-checks.yml index da7c77c34a133..65e1a251ebb56 100644 --- a/.github/workflows/code-checks.yml +++ b/.github/workflows/code-checks.yml @@ -33,9 +33,11 @@ jobs: - name: Run pre-commit uses: pre-commit/action@v2.0.3 + with: + extra_args: --hook-stage manual --all-files - typing_and_docstring_validation: - name: Docstring and typing validation + docstring_validation: + name: Docstring validation runs-on: ubuntu-latest defaults: run: @@ -87,18 +89,6 @@ jobs: run: ci/code_checks.sh docstrings if: ${{ steps.build.outcome == 'success' }} - - name: Run typing validation (mypy) - uses: pre-commit/action@v2.0.3 - with: - extra_args: --hook-stage manual --all-files mypy - if: ${{ steps.build.outcome == 'success' }} - - - name: Run typing validation (pyright) - uses: pre-commit/action@v2.0.3 - with: - extra_args: --hook-stage manual --all-files pyright - if: ${{ steps.build.outcome == 'success' }} - - name: Run docstring validation script tests run: pytest scripts if: ${{ steps.build.outcome == 'success' }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fc92119ca4160..15fd69d30da36 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -81,25 +81,45 @@ repos: additional_dependencies: *flake8_dependencies - repo: local hooks: - - id: pyright - name: pyright - entry: pyright - # note: assumes python env is setup and activated - language: node + - id: mypy + name: mypy + entry: mypy + language: python pass_filenames: false types: [python] - stages: [manual] - additional_dependencies: ['pyright@1.1.247'] + additional_dependencies: + - mypy==0.950 + - numpy>=1.18.5 + - pytest>=6.0 + - matplotlib>=3.3.2 + - scipy>=1.4.1 + - xlwt + - odfpy + - pyarrow>2.0.1 + - botocore>=1.11 + - numexpr>=2.7.1 + - fsspec>=0.7.4 + - hypothesis>=5.5.3 + - xlrd + - ipython>=7.11.1 + - lxml + - types-PyMySQL + - types-python-dateutil + - types-pytz + - types-setuptools + - tables>=3.6.1 + - xlsxwriter + - jinja2 - repo: local hooks: - - id: mypy - name: mypy - entry: mypy - # note: assumes python env is setup and activated - language: system + - id: pyright + name: pyright + entry: ./run_pyright.sh + language: node pass_filenames: false types: [python] stages: [manual] + additional_dependencies: ['pyright@1.1.247'] - repo: local hooks: - id: flake8-rst diff --git a/run_pyright.sh b/run_pyright.sh new file mode 100755 index 0000000000000..30ebfc10826d9 --- /dev/null +++ b/run_pyright.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# run pyright in mypy's pre-commit environment + +# find mypy's environment +mypy_path=$(find ~/.cache/pre-commit/ -type f -name "mypy") + +# remove "bin/mypy" +mypy_env=$(dirname $(dirname $mypy_path)) + +env_name=$(basename $mypy_env) +env_folder=$(dirname $mypy_env) + +# the environment name needs to be specified in the config file +sed -i "s/tool.pyright\]/tool.pyright\]\nvenv = \"$env_name\"/" pyproject.toml + +# the folder can be specified as an argument +RET=0 +pyright --venv-path $env_folder +RET=$(($RET + $?)) + +# revert sed +git checkout pyproject.toml + +exit $RET From d2491773627215eee9013c32ceb7eeb1dc0bd7e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torsten=20W=C3=B6rtwein?= Date: Fri, 20 May 2022 17:11:06 -0400 Subject: [PATCH 3/5] Revert "move mypy+pyright to pre-commit" --- .github/workflows/code-checks.yml | 18 ++++++++++--- .pre-commit-config.yaml | 44 +++++++++---------------------- run_pyright.sh | 25 ------------------ 3 files changed, 26 insertions(+), 61 deletions(-) delete mode 100755 run_pyright.sh diff --git a/.github/workflows/code-checks.yml b/.github/workflows/code-checks.yml index 65e1a251ebb56..da7c77c34a133 100644 --- a/.github/workflows/code-checks.yml +++ b/.github/workflows/code-checks.yml @@ -33,11 +33,9 @@ jobs: - name: Run pre-commit uses: pre-commit/action@v2.0.3 - with: - extra_args: --hook-stage manual --all-files - docstring_validation: - name: Docstring validation + typing_and_docstring_validation: + name: Docstring and typing validation runs-on: ubuntu-latest defaults: run: @@ -89,6 +87,18 @@ jobs: run: ci/code_checks.sh docstrings if: ${{ steps.build.outcome == 'success' }} + - name: Run typing validation (mypy) + uses: pre-commit/action@v2.0.3 + with: + extra_args: --hook-stage manual --all-files mypy + if: ${{ steps.build.outcome == 'success' }} + + - name: Run typing validation (pyright) + uses: pre-commit/action@v2.0.3 + with: + extra_args: --hook-stage manual --all-files pyright + if: ${{ steps.build.outcome == 'success' }} + - name: Run docstring validation script tests run: pytest scripts if: ${{ steps.build.outcome == 'success' }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 15fd69d30da36..fc92119ca4160 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -79,47 +79,27 @@ repos: hooks: - id: yesqa additional_dependencies: *flake8_dependencies -- repo: local - hooks: - - id: mypy - name: mypy - entry: mypy - language: python - pass_filenames: false - types: [python] - additional_dependencies: - - mypy==0.950 - - numpy>=1.18.5 - - pytest>=6.0 - - matplotlib>=3.3.2 - - scipy>=1.4.1 - - xlwt - - odfpy - - pyarrow>2.0.1 - - botocore>=1.11 - - numexpr>=2.7.1 - - fsspec>=0.7.4 - - hypothesis>=5.5.3 - - xlrd - - ipython>=7.11.1 - - lxml - - types-PyMySQL - - types-python-dateutil - - types-pytz - - types-setuptools - - tables>=3.6.1 - - xlsxwriter - - jinja2 - repo: local hooks: - id: pyright name: pyright - entry: ./run_pyright.sh + entry: pyright + # note: assumes python env is setup and activated language: node pass_filenames: false types: [python] stages: [manual] additional_dependencies: ['pyright@1.1.247'] +- repo: local + hooks: + - id: mypy + name: mypy + entry: mypy + # note: assumes python env is setup and activated + language: system + pass_filenames: false + types: [python] + stages: [manual] - repo: local hooks: - id: flake8-rst diff --git a/run_pyright.sh b/run_pyright.sh deleted file mode 100755 index 30ebfc10826d9..0000000000000 --- a/run_pyright.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -# run pyright in mypy's pre-commit environment - -# find mypy's environment -mypy_path=$(find ~/.cache/pre-commit/ -type f -name "mypy") - -# remove "bin/mypy" -mypy_env=$(dirname $(dirname $mypy_path)) - -env_name=$(basename $mypy_env) -env_folder=$(dirname $mypy_env) - -# the environment name needs to be specified in the config file -sed -i "s/tool.pyright\]/tool.pyright\]\nvenv = \"$env_name\"/" pyproject.toml - -# the folder can be specified as an argument -RET=0 -pyright --venv-path $env_folder -RET=$(($RET + $?)) - -# revert sed -git checkout pyproject.toml - -exit $RET From a8ad73fc761ff29fbbcf49dd18bb3a5b5b1bf4a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torsten=20W=C3=B6rtwein?= Date: Fri, 20 May 2022 17:41:53 -0400 Subject: [PATCH 4/5] system+manual --- .github/workflows/code-checks.yml | 14 +++++------ .pre-commit-config.yaml | 6 ++++- ci/code_checks.sh | 23 ++----------------- .../development/contributing_codebase.rst | 14 +++++------ environment.yml | 2 +- requirements-dev.txt | 2 +- 6 files changed, 23 insertions(+), 38 deletions(-) diff --git a/.github/workflows/code-checks.yml b/.github/workflows/code-checks.yml index da7c77c34a133..7812c4dd00bcb 100644 --- a/.github/workflows/code-checks.yml +++ b/.github/workflows/code-checks.yml @@ -87,16 +87,16 @@ jobs: run: ci/code_checks.sh docstrings if: ${{ steps.build.outcome == 'success' }} - - name: Run typing validation (mypy) - uses: pre-commit/action@v2.0.3 - with: - extra_args: --hook-stage manual --all-files mypy - if: ${{ steps.build.outcome == 'success' }} + - name: Use existing environment for type checking + run: | + echo $PATH >> $GITHUB_PATH + echo "PYTHONHOME=$PYTHONHOME" >> $GITHUB_ENV + echo "PYTHONPATH=$PYTHONPATH" >> $GITHUB_ENV - - name: Run typing validation (pyright) + - name: Typing uses: pre-commit/action@v2.0.3 with: - extra_args: --hook-stage manual --all-files pyright + extra_args: --hook-stage manual --all-files if: ${{ steps.build.outcome == 'success' }} - name: Run docstring validation script tests diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fc92119ca4160..ee8889d22f360 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,5 +1,7 @@ minimum_pre_commit_version: 2.9.2 exclude: ^LICENSES/|\.(html|csv|svg)$ +# reserve "manual" for mypy and pyright +default_stages: [commit, merge-commit, push, prepare-commit-msg, commit-msg, post-checkout, post-commit, post-merge, post-rewrite] ci: autofix_prs: false repos: @@ -31,7 +33,9 @@ repos: - id: debug-statements - id: end-of-file-fixer exclude: \.txt$ + stages: [commit, merge-commit, push, prepare-commit-msg, commit-msg, post-checkout, post-commit, post-merge, post-rewrite] - id: trailing-whitespace + stages: [commit, merge-commit, push, prepare-commit-msg, commit-msg, post-checkout, post-commit, post-merge, post-rewrite] - repo: https://github.com/cpplint/cpplint rev: 1.6.0 hooks: @@ -89,7 +93,7 @@ repos: pass_filenames: false types: [python] stages: [manual] - additional_dependencies: ['pyright@1.1.247'] + additional_dependencies: ['pyright@1.1.248'] - repo: local hooks: - id: mypy diff --git a/ci/code_checks.sh b/ci/code_checks.sh index d7e685f8d055f..b42ffc66f7714 100755 --- a/ci/code_checks.sh +++ b/ci/code_checks.sh @@ -11,11 +11,10 @@ # $ ./ci/code_checks.sh code # checks on imported code # $ ./ci/code_checks.sh doctests # run doctests # $ ./ci/code_checks.sh docstrings # validate docstring errors -# $ ./ci/code_checks.sh typing # run static type analysis # $ ./ci/code_checks.sh single-docs # check single-page docs build warning-free -[[ -z "$1" || "$1" == "code" || "$1" == "doctests" || "$1" == "docstrings" || "$1" == "typing" || "$1" == "single-docs" ]] || \ - { echo "Unknown command $1. Usage: $0 [code|doctests|docstrings|typing]"; exit 9999; } +[[ -z "$1" || "$1" == "code" || "$1" == "doctests" || "$1" == "docstrings" || "$1" == "single-docs" ]] || \ + { echo "Unknown command $1. Usage: $0 [code|doctests|docstrings]"; exit 9999; } BASE_DIR="$(dirname $0)/.." RET=0 @@ -85,24 +84,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then fi -### TYPING ### -if [[ -z "$CHECK" || "$CHECK" == "typing" ]]; then - - echo "mypy --version" - mypy --version - - MSG='Performing static analysis using mypy' ; echo $MSG - mypy - RET=$(($RET + $?)) ; echo $MSG "DONE" - - # run pyright, if it is installed - if command -v pyright &> /dev/null ; then - MSG='Performing static analysis using pyright' ; echo $MSG - pyright - RET=$(($RET + $?)) ; echo $MSG "DONE" - fi -fi - ### SINGLE-PAGE DOCS ### if [[ -z "$CHECK" || "$CHECK" == "single-docs" ]]; then python doc/make.py --warnings-are-errors --single pandas.Series.value_counts diff --git a/doc/source/development/contributing_codebase.rst b/doc/source/development/contributing_codebase.rst index fcaa8adcdcae9..3437ddfbffdcf 100644 --- a/doc/source/development/contributing_codebase.rst +++ b/doc/source/development/contributing_codebase.rst @@ -23,9 +23,9 @@ contributing them to the project:: ./ci/code_checks.sh -The script validates the doctests, formatting in docstrings, static typing, and +The script validates the doctests, formatting in docstrings, and imported modules. It is possible to run the checks independently by using the -parameters ``docstring``, ``code``, ``typing``, and ``doctests`` +parameters ``docstring``, ``code``, and ``doctests`` (e.g. ``./ci/code_checks.sh doctests``). In addition, because a lot of people use our library, it is important that we @@ -33,9 +33,9 @@ do not make sudden changes to the code that could have the potential to break a lot of user code as a result, that is, we need it to be as *backwards compatible* as possible to avoid mass breakages. -In addition to ``./ci/code_checks.sh``, some extra checks are run by -``pre-commit`` - see :ref:`here ` for how to -run them. +In addition to ``./ci/code_checks.sh``, some extra checks (including static type +checking) are run by ``pre-commit`` - see :ref:`here ` +for how to run them. .. _contributing.pre-commit: @@ -260,9 +260,9 @@ pandas uses `mypy `_ and `pyright =1.21.0) is required for type validation. +in your activated python environment. A recent version of ``numpy`` (>=1.22.0) is required for type validation. .. _contributing.ci: diff --git a/environment.yml b/environment.yml index df69e654018fb..7c08fe2901086 100644 --- a/environment.yml +++ b/environment.yml @@ -25,7 +25,7 @@ dependencies: - flake8-comprehensions=3.7.0 # used by flake8, linting of unnecessary comprehensions - isort>=5.2.1 # check that imports are in the right order - mypy=0.950 - - pre-commit>=2.9.2 + - pre-commit>=2.15.0 - pycodestyle # used by flake8 - pyupgrade diff --git a/requirements-dev.txt b/requirements-dev.txt index 36d3720df34d8..f13da7709c157 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -13,7 +13,7 @@ flake8-bugbear==21.3.2 flake8-comprehensions==3.7.0 isort>=5.2.1 mypy==0.950 -pre-commit>=2.9.2 +pre-commit>=2.15.0 pycodestyle pyupgrade gitpython From bc3b01aeaf78dd738060fefb2cfcc83f20cc0cc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torsten=20W=C3=B6rtwein?= Date: Sun, 22 May 2022 10:33:13 -0400 Subject: [PATCH 5/5] Update .pre-commit-config.yaml Co-authored-by: Marco Edward Gorelli --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ee8889d22f360..81c76e02c73f8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,4 +1,4 @@ -minimum_pre_commit_version: 2.9.2 +minimum_pre_commit_version: 2.15.0 exclude: ^LICENSES/|\.(html|csv|svg)$ # reserve "manual" for mypy and pyright default_stages: [commit, merge-commit, push, prepare-commit-msg, commit-msg, post-checkout, post-commit, post-merge, post-rewrite]