Merge remote-tracking branch 'upstream/main' into internals-column-setitem

Author: jorisvandenbossche

.github/workflows/code-checks.yml

@@ -67,15 +67,6 @@ jobs:
         environment-file: ${{ env.ENV_FILE }}
         use-only-tar-bz2: true
 
-    - name: Install node.js (for pyright)
-      uses: actions/setup-node@v3
-      with:
-        node-version: "16"
-
-    - name: Install pyright
-      # note: keep version in sync with .pre-commit-config.yaml
-      run: npm install -g [email protected]
-
     - name: Build Pandas
       id: build
       uses: ./.github/actions/build_pandas
@@ -96,8 +87,16 @@ jobs:
       run: ci/code_checks.sh docstrings
       if: ${{ steps.build.outcome == 'success' }}
 
-    - name: Run typing validation
-      run: ci/code_checks.sh typing
+    - name: Use existing environment for type checking
+      run: |
+        echo $PATH >> $GITHUB_PATH
+        echo "PYTHONHOME=$PYTHONHOME" >> $GITHUB_ENV
+        echo "PYTHONPATH=$PYTHONPATH" >> $GITHUB_ENV
+
+    - name: Typing
+      uses: pre-commit/[email protected]
+      with:
+        extra_args: --hook-stage manual --all-files
       if: ${{ steps.build.outcome == 'success' }}
 
     - name: Run docstring validation script tests

.pre-commit-config.yaml

@@ -1,5 +1,7 @@
-minimum_pre_commit_version: 2.9.2
+minimum_pre_commit_version: 2.15.0
 exclude: ^LICENSES/|\.(html|csv|svg)$
+# reserve "manual" for mypy and pyright
+default_stages: [commit, merge-commit, push, prepare-commit-msg, commit-msg, post-checkout, post-commit, post-merge, post-rewrite]
 ci:
     autofix_prs: false
 repos:
@@ -31,7 +33,9 @@ repos:
     -   id: debug-statements
     -   id: end-of-file-fixer
         exclude: \.txt$
+        stages: [commit, merge-commit, push, prepare-commit-msg, commit-msg, post-checkout, post-commit, post-merge, post-rewrite]
     -   id: trailing-whitespace
+        stages: [commit, merge-commit, push, prepare-commit-msg, commit-msg, post-checkout, post-commit, post-merge, post-rewrite]
 -   repo: https://github.com/cpplint/cpplint
     rev: 1.6.0
     hooks:
@@ -84,12 +88,22 @@ repos:
     -   id: pyright
         name: pyright
         entry: pyright
+        # note: assumes python env is setup and activated
         language: node
         pass_filenames: false
         types: [python]
         stages: [manual]
-        # note: keep version in sync with .github/workflows/code-checks.yml
-        additional_dependencies: ['[email protected]']
+        additional_dependencies: ['[email protected]']
+-   repo: local
+    hooks:
+    -   id: mypy
+        name: mypy
+        entry: mypy
+        # note: assumes python env is setup and activated
+        language: system
+        pass_filenames: false
+        types: [python]
+        stages: [manual]
 -   repo: local
     hooks:
     -   id: flake8-rst

asv_bench/benchmarks/arithmetic.py

@@ -420,7 +420,7 @@ def time_add_overflow_both_arg_nan(self):
 
 
 hcal = pd.tseries.holiday.USFederalHolidayCalendar()
-# These offsets currently raise a NotImplimentedError with .apply_index()
+# These offsets currently raise a NotImplementedError with .apply_index()
 non_apply = [
     pd.offsets.Day(),
     pd.offsets.BYearEnd(),

asv_bench/benchmarks/tslibs/normalize.py

@@ -36,8 +36,10 @@ def setup(self, size, tz):
             raise NotImplementedError
 
     def time_normalize_i8_timestamps(self, size, tz):
-        normalize_i8_timestamps(self.i8data, tz)
+        # 10 i.e. NPY_FR_ns
+        normalize_i8_timestamps(self.i8data, tz, 10)
 
     def time_is_date_array_normalized(self, size, tz):
         # TODO: cases with different levels of short-circuiting
-        is_date_array_normalized(self.i8data, tz)
+        # 10 i.e. NPY_FR_ns
+        is_date_array_normalized(self.i8data, tz, 10)

asv_bench/benchmarks/tslibs/offsets.py

@@ -14,7 +14,7 @@
     pass
 
 hcal = pandas.tseries.holiday.USFederalHolidayCalendar()
-# These offsets currently raise a NotImplimentedError with .apply_index()
+# These offsets currently raise a NotImplementedError with .apply_index()
 non_apply = [
     offsets.Day(),
     offsets.BYearEnd(),

ci/code_checks.sh

@@ -11,11 +11,10 @@
 #   $ ./ci/code_checks.sh code          # checks on imported code
 #   $ ./ci/code_checks.sh doctests      # run doctests
 #   $ ./ci/code_checks.sh docstrings    # validate docstring errors
-#   $ ./ci/code_checks.sh typing        # run static type analysis
 #   $ ./ci/code_checks.sh single-docs   # check single-page docs build warning-free
 
-[[ -z "$1" || "$1" == "code" || "$1" == "doctests" || "$1" == "docstrings" || "$1" == "typing" || "$1" == "single-docs" ]] || \
-    { echo "Unknown command $1. Usage: $0 [code|doctests|docstrings|typing]"; exit 9999; }
+[[ -z "$1" || "$1" == "code" || "$1" == "doctests" || "$1" == "docstrings" || "$1" == "single-docs" ]] || \
+    { echo "Unknown command $1. Usage: $0 [code|doctests|docstrings]"; exit 9999; }
 
 BASE_DIR="$(dirname $0)/.."
 RET=0
@@ -85,24 +84,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
 
 fi
 
-### TYPING ###
-if [[ -z "$CHECK" || "$CHECK" == "typing" ]]; then
-
-    echo "mypy --version"
-    mypy --version
-
-    MSG='Performing static analysis using mypy' ; echo $MSG
-    mypy
-    RET=$(($RET + $?)) ; echo $MSG "DONE"
-
-    # run pyright, if it is installed
-    if command -v pyright &> /dev/null ; then
-        MSG='Performing static analysis using pyright' ; echo $MSG
-        pyright
-        RET=$(($RET + $?)) ; echo $MSG "DONE"
-    fi
-fi
-
 ### SINGLE-PAGE DOCS ###
 if [[ -z "$CHECK" || "$CHECK" == "single-docs" ]]; then
     python doc/make.py --warnings-are-errors --single pandas.Series.value_counts

doc/source/development/contributing_codebase.rst

@@ -23,19 +23,19 @@ contributing them to the project::
 
    ./ci/code_checks.sh
 
-The script validates the doctests, formatting in docstrings, static typing, and
+The script validates the doctests, formatting in docstrings, and
 imported modules. It is possible to run the checks independently by using the
-parameters ``docstring``, ``code``, ``typing``, and ``doctests``
+parameters ``docstring``, ``code``, and ``doctests``
 (e.g. ``./ci/code_checks.sh doctests``).
 
 In addition, because a lot of people use our library, it is important that we
 do not make sudden changes to the code that could have the potential to break
 a lot of user code as a result, that is, we need it to be as *backwards compatible*
 as possible to avoid mass breakages.
 
-In addition to ``./ci/code_checks.sh``, some extra checks are run by
-``pre-commit`` - see :ref:`here <contributing.pre-commit>` for how to
-run them.
+In addition to ``./ci/code_checks.sh``, some extra checks (including static type
+checking) are run by ``pre-commit`` - see :ref:`here <contributing.pre-commit>`
+for how to run them.
 
 .. _contributing.pre-commit:
 
@@ -260,9 +260,9 @@ pandas uses `mypy <http://mypy-lang.org>`_ and `pyright <https://github.com/micr
 
 .. code-block:: shell
 
-   ./ci/code_checks.sh typing
+   pre-commit run --hook-stage manual --all-files
 
-A recent version of ``numpy`` (>=1.21.0) is required for type validation.
+in your activated python environment. A recent version of ``numpy`` (>=1.22.0) is required for type validation.
 
 .. _contributing.ci:
 

environment.yml

@@ -25,7 +25,7 @@ dependencies:
   - flake8-comprehensions=3.7.0  # used by flake8, linting of unnecessary comprehensions
   - isort>=5.2.1  # check that imports are in the right order
   - mypy=0.950
-  - pre-commit>=2.9.2
+  - pre-commit>=2.15.0
   - pycodestyle  # used by flake8
   - pyupgrade