From 8197c6163c96a531fbbcecec0e08f2b5760fb78f Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Sat, 19 Sep 2020 08:05:46 +0100 Subject: [PATCH 01/11] add CI pre-commit check, include pyupgrade, remove unnecessary 'language' --- .github/workflows/pre-commit.yml | 14 ++++++++++++++ .pre-commit-config.yaml | 10 +++++----- 2 files changed, 19 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/pre-commit.yml diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml new file mode 100644 index 0000000000000..723347913ac38 --- /dev/null +++ b/.github/workflows/pre-commit.yml @@ -0,0 +1,14 @@ +name: pre-commit + +on: + pull_request: + push: + branches: [master] + +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - uses: pre-commit/action@v2.0.0 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 309e22e71a523..dad48688dcf2c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,23 +3,19 @@ repos: rev: 19.10b0 hooks: - id: black - language_version: python3 - repo: https://gitlab.com/pycqa/flake8 rev: 3.8.3 hooks: - id: flake8 - language: python_venv additional_dependencies: [flake8-comprehensions>=3.1.0] - id: flake8 name: flake8-pyx - language: python_venv files: \.(pyx|pxd)$ types: - file args: [--append-config=flake8/cython.cfg] - id: flake8 name: flake8-pxd - language: python_venv files: \.pxi\.in$ types: - file @@ -28,5 +24,9 @@ repos: rev: v5.2.2 hooks: - id: isort - language: python_venv exclude: ^pandas/__init__\.py$|^pandas/core/api\.py$ +- repo: https://github.com/asottile/pyupgrade + rev: v2.7.2 + hooks: + - id: pyupgrade + args: [--py37-plus] From 1a6b47758f47d59b0e7739b53b67867c1fa25f8b Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Sat, 19 Sep 2020 08:07:54 +0100 Subject: [PATCH 02/11] upgrade doc/sphinxext/announce.py --- doc/sphinxext/announce.py | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/sphinxext/announce.py b/doc/sphinxext/announce.py index 9c175e4e58b45..2ec0b515ea95c 100755 --- a/doc/sphinxext/announce.py +++ b/doc/sphinxext/announce.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- encoding:utf-8 -*- """ Script to generate contributor and pull request lists From e8135b46709b50fd10ff53e48847108d27f336ee Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Sat, 19 Sep 2020 08:09:40 +0100 Subject: [PATCH 03/11] remove doubled checks --- ci/code_checks.sh | 45 --------------------------------------------- 1 file changed, 45 deletions(-) diff --git a/ci/code_checks.sh b/ci/code_checks.sh index 54aa830379c07..b3ce3195aaf5e 100755 --- a/ci/code_checks.sh +++ b/ci/code_checks.sh @@ -48,38 +48,6 @@ fi ### LINTING ### if [[ -z "$CHECK" || "$CHECK" == "lint" ]]; then - echo "black --version" - black --version - - MSG='Checking black formatting' ; echo $MSG - black . --check - RET=$(($RET + $?)) ; echo $MSG "DONE" - - # `setup.cfg` contains the list of error codes that are being ignored in flake8 - - echo "flake8 --version" - flake8 --version - - # pandas/_libs/src is C code, so no need to search there. - MSG='Linting .py code' ; echo $MSG - flake8 --format="$FLAKE8_FORMAT" . - RET=$(($RET + $?)) ; echo $MSG "DONE" - - MSG='Linting .pyx and .pxd code' ; echo $MSG - flake8 --format="$FLAKE8_FORMAT" pandas --append-config=flake8/cython.cfg - RET=$(($RET + $?)) ; echo $MSG "DONE" - - MSG='Linting .pxi.in' ; echo $MSG - flake8 --format="$FLAKE8_FORMAT" pandas/_libs --append-config=flake8/cython-template.cfg - RET=$(($RET + $?)) ; echo $MSG "DONE" - - echo "flake8-rst --version" - flake8-rst --version - - MSG='Linting code-blocks in .rst documentation' ; echo $MSG - flake8-rst doc/source --filename=*.rst --format="$FLAKE8_FORMAT" - RET=$(($RET + $?)) ; echo $MSG "DONE" - # Check that cython casting is of the form `obj` as opposed to ` obj`; # it doesn't make a difference, but we want to be internally consistent. # Note: this grep pattern is (intended to be) equivalent to the python @@ -132,19 +100,6 @@ if [[ -z "$CHECK" || "$CHECK" == "lint" ]]; then fi RET=$(($RET + $?)) ; echo $MSG "DONE" - echo "isort --version-number" - isort --version-number - - # Imports - Check formatting using isort see setup.cfg for settings - MSG='Check import format using isort' ; echo $MSG - ISORT_CMD="isort --quiet --check-only pandas asv_bench scripts web" - if [[ "$GITHUB_ACTIONS" == "true" ]]; then - eval $ISORT_CMD | awk '{print "##[error]" $0}'; RET=$(($RET + ${PIPESTATUS[0]})) - else - eval $ISORT_CMD - fi - RET=$(($RET + $?)) ; echo $MSG "DONE" - fi ### PATTERNS ### From f4788fad3ff8f960e22d61bda6c2dd2d766fc1e8 Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Sat, 19 Sep 2020 08:19:23 +0100 Subject: [PATCH 04/11] update deprecated repo --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index dad48688dcf2c..2fa5f5ec5d4fa 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -20,8 +20,8 @@ repos: types: - file args: [--append-config=flake8/cython-template.cfg] -- repo: https://github.com/pre-commit/mirrors-isort - rev: v5.2.2 +- repo: https://github.com/PyCQA/isort + rev: 5.2.2 hooks: - id: isort exclude: ^pandas/__init__\.py$|^pandas/core/api\.py$ From 27e285e363b27f2fc3e0529bdc5051120d2996b8 Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Sat, 19 Sep 2020 08:24:56 +0100 Subject: [PATCH 05/11] add flake8-rst back to code_checks --- ci/code_checks.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ci/code_checks.sh b/ci/code_checks.sh index b3ce3195aaf5e..13bc935a7d599 100755 --- a/ci/code_checks.sh +++ b/ci/code_checks.sh @@ -48,6 +48,13 @@ fi ### LINTING ### if [[ -z "$CHECK" || "$CHECK" == "lint" ]]; then + echo "flake8-rst --version" + flake8-rst --version + + MSG='Linting code-blocks in .rst documentation' ; echo $MSG + flake8-rst doc/source --filename=*.rst --format="$FLAKE8_FORMAT" + RET=$(($RET + $?)) ; echo $MSG "DONE" + # Check that cython casting is of the form `obj` as opposed to ` obj`; # it doesn't make a difference, but we want to be internally consistent. # Note: this grep pattern is (intended to be) equivalent to the python From 1e5bc8063869df632f2d317354a88eae2dc0ad43 Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Mon, 21 Sep 2020 13:44:57 +0100 Subject: [PATCH 06/11] don't remove from code checks yet --- ci/code_checks.sh | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/ci/code_checks.sh b/ci/code_checks.sh index 13bc935a7d599..54aa830379c07 100755 --- a/ci/code_checks.sh +++ b/ci/code_checks.sh @@ -48,6 +48,31 @@ fi ### LINTING ### if [[ -z "$CHECK" || "$CHECK" == "lint" ]]; then + echo "black --version" + black --version + + MSG='Checking black formatting' ; echo $MSG + black . --check + RET=$(($RET + $?)) ; echo $MSG "DONE" + + # `setup.cfg` contains the list of error codes that are being ignored in flake8 + + echo "flake8 --version" + flake8 --version + + # pandas/_libs/src is C code, so no need to search there. + MSG='Linting .py code' ; echo $MSG + flake8 --format="$FLAKE8_FORMAT" . + RET=$(($RET + $?)) ; echo $MSG "DONE" + + MSG='Linting .pyx and .pxd code' ; echo $MSG + flake8 --format="$FLAKE8_FORMAT" pandas --append-config=flake8/cython.cfg + RET=$(($RET + $?)) ; echo $MSG "DONE" + + MSG='Linting .pxi.in' ; echo $MSG + flake8 --format="$FLAKE8_FORMAT" pandas/_libs --append-config=flake8/cython-template.cfg + RET=$(($RET + $?)) ; echo $MSG "DONE" + echo "flake8-rst --version" flake8-rst --version @@ -107,6 +132,19 @@ if [[ -z "$CHECK" || "$CHECK" == "lint" ]]; then fi RET=$(($RET + $?)) ; echo $MSG "DONE" + echo "isort --version-number" + isort --version-number + + # Imports - Check formatting using isort see setup.cfg for settings + MSG='Check import format using isort' ; echo $MSG + ISORT_CMD="isort --quiet --check-only pandas asv_bench scripts web" + if [[ "$GITHUB_ACTIONS" == "true" ]]; then + eval $ISORT_CMD | awk '{print "##[error]" $0}'; RET=$(($RET + ${PIPESTATUS[0]})) + else + eval $ISORT_CMD + fi + RET=$(($RET + $?)) ; echo $MSG "DONE" + fi ### PATTERNS ### From 9c98406152f591e5ba559bf0b274a43c7a0f84f0 Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Mon, 21 Sep 2020 18:42:33 +0100 Subject: [PATCH 07/11] :memo: update code standards --- doc/source/development/contributing.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/doc/source/development/contributing.rst b/doc/source/development/contributing.rst index e5c6f77eea3ef..411f0437e1ceb 100644 --- a/doc/source/development/contributing.rst +++ b/doc/source/development/contributing.rst @@ -634,6 +634,10 @@ do not make sudden changes to the code that could have the potential to break a lot of user code as a result, that is, we need it to be as *backwards compatible* as possible to avoid mass breakages. +In addition to `./ci/code_checks.sh`, some extra checks are run by +``pre-commit`` - see :ref:`here ` for how to +run them. + Additional standards are outlined on the :ref:`pandas code style guide ` Optional dependencies @@ -826,6 +830,13 @@ remain up-to-date with our code checks as they change. Note that if needed, you can skip these checks with ``git commit --no-verify``. +If you don't want to use ``pre-commit`` as part of your workflow, you can still use it +to run its checks by running:: + + pre-commit run --files + +without having to have done ``pre-commit install`` beforehand. + Backwards compatibility ~~~~~~~~~~~~~~~~~~~~~~~ From 635b8d7c406f229bcfcba2663be854d243438aab Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Mon, 21 Sep 2020 18:44:01 +0100 Subject: [PATCH 08/11] double-backtick --- doc/source/development/contributing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/development/contributing.rst b/doc/source/development/contributing.rst index 411f0437e1ceb..6f29ea8b764bd 100644 --- a/doc/source/development/contributing.rst +++ b/doc/source/development/contributing.rst @@ -634,7 +634,7 @@ do not make sudden changes to the code that could have the potential to break a lot of user code as a result, that is, we need it to be as *backwards compatible* as possible to avoid mass breakages. -In addition to `./ci/code_checks.sh`, some extra checks are run by +In addition to ``./ci/code_checks.sh``, some extra checks are run by ``pre-commit`` - see :ref:`here ` for how to run them. From e2a334aabb87cf77c8906ee3ebdf18f6349c1f28 Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Mon, 21 Sep 2020 18:46:02 +0100 Subject: [PATCH 09/11] :heavy_plus_sign: add pyupgrade to environment.yml --- environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environment.yml b/environment.yml index 36bbd3d307159..534d429ab939e 100644 --- a/environment.yml +++ b/environment.yml @@ -23,6 +23,7 @@ dependencies: - isort>=5.2.1 # check that imports are in the right order - mypy=0.782 - pycodestyle # used by flake8 + - pyupgrade # documentation - gitpython # obtain contributors from git for whatsnew From b8aea466df7ae67fef956e71267579b49077c8a6 Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Mon, 21 Sep 2020 18:47:33 +0100 Subject: [PATCH 10/11] :heavy_plus_sign: add pre-commit to environment.yml --- environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environment.yml b/environment.yml index 534d429ab939e..1a260b0f7cc56 100644 --- a/environment.yml +++ b/environment.yml @@ -22,6 +22,7 @@ dependencies: - flake8-rst>=0.6.0,<=0.7.0 # linting of code blocks in rst files - isort>=5.2.1 # check that imports are in the right order - mypy=0.782 + - pre-commit - pycodestyle # used by flake8 - pyupgrade From 5b6faa102e81b21b2ec94bef8fa3ddbe33a78ab6 Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Tue, 22 Sep 2020 08:28:05 +0100 Subject: [PATCH 11/11] generate pip reqs from conda --- requirements-dev.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements-dev.txt b/requirements-dev.txt index fb647c10f72bc..9ee26562e6d4b 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -13,7 +13,9 @@ flake8-comprehensions>=3.1.0 flake8-rst>=0.6.0,<=0.7.0 isort>=5.2.1 mypy==0.782 +pre-commit pycodestyle +pyupgrade gitpython gitdb sphinx