Merge remote-tracking branch 'upstream/main' into uncertainties

Needed in order to fix broken reduce operations (pandas-dev#52788).

Signed-off-by: Michael Tiemann <[email protected]>

Author: MichaelTiemannOSC

.github/CODEOWNERS

@@ -9,6 +9,7 @@ web/                              @datapythonista
 
 # docs
 doc/cheatsheet                    @Dr-Irv
+doc/source/development            @noatamir
 
 # pandas
 pandas/_libs/                     @WillAyd

.github/actions/build_pandas/action.yml

@@ -12,12 +12,13 @@ runs:
       run: |
         micromamba info
         micromamba list
+        pip list --pre
       shell: bash -el {0}
 
     - name: Uninstall existing Pandas installation
       run: |
-        if pip list | grep -q ^pandas; then
-          pip uninstall -y pandas || true
+        if pip show pandas 1>/dev/null; then
+          pip uninstall -y pandas
         fi
       shell: bash -el {0}
 

.github/workflows/cache-cleanup-weekly.yml

@@ -0,0 +1,32 @@
+name: Purge caches once a week
+on:
+  schedule:
+    # 4:10 UTC on Sunday
+    - cron: "10 4 * * 0"
+
+jobs:
+  cleanup:
+    runs-on: ubuntu-latest
+    if: github.repository_owner == 'pandas-dev'
+    permissions:
+      actions: write
+    steps:
+      - name: Clean Cache
+        run: |
+          gh extension install actions/gh-actions-cache
+
+          REPO=${{ github.repository }}
+
+          echo "Fetching list of cache key"
+          allCaches=$(gh actions-cache list -L 100 -R $REPO | cut -f 1 )
+
+          ## Setting this to not fail the workflow while deleting cache keys.
+          set +e
+          echo "Deleting caches..."
+          for cacheKey in $allCaches
+          do
+              gh actions-cache delete $cacheKey -R $REPO --confirm
+          done
+          echo "Done"
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/unit-tests.yml

@@ -57,7 +57,15 @@ jobs:
             # Also install zh_CN (its encoding is gb2312) but do not activate it.
             # It will be temporarily activated during tests with locale.setlocale
             extra_loc: "zh_CN"
-          - name: "Copy-on-Write"
+          - name: "Copy-on-Write 3.9"
+            env_file: actions-39.yaml
+            pattern: "not slow and not network and not single_cpu"
+            pandas_copy_on_write: "1"
+          - name: "Copy-on-Write 3.10"
+            env_file: actions-310.yaml
+            pattern: "not slow and not network and not single_cpu"
+            pandas_copy_on_write: "1"
+          - name: "Copy-on-Write 3.11"
             env_file: actions-311.yaml
             pattern: "not slow and not network and not single_cpu"
             pandas_copy_on_write: "1"
@@ -165,7 +173,7 @@ jobs:
       uses: ./.github/actions/run-tests
       env:
         PATTERN: 'single_cpu'
-        PYTEST_WORKERS: 1
+        PYTEST_WORKERS: 0
       if: ${{ matrix.pattern == '' && (always() && steps.build.outcome == 'success')}}
 
   macos-windows:
@@ -185,8 +193,8 @@ jobs:
       PANDAS_CI: 1
       PYTEST_TARGET: pandas
       PATTERN: "not slow and not db and not network and not single_cpu"
-      # GH 47443: PYTEST_WORKERS > 1 crashes Windows builds with memory related errors
-      PYTEST_WORKERS: ${{ matrix.os == 'macos-latest' && 'auto' || '1' }}
+      # GH 47443: PYTEST_WORKERS > 0 crashes Windows builds with memory related errors
+      PYTEST_WORKERS: ${{ matrix.os == 'macos-latest' && 'auto' || '0' }}
 
     steps:
       - name: Checkout

.github/workflows/wheels.yml

@@ -110,7 +110,7 @@ jobs:
           path: ./dist
 
       - name: Build wheels
-        uses: pypa/cibuildwheel@v2.13.1
+        uses: pypa/cibuildwheel@v2.14.1
         # TODO: Build wheels from sdist again
         # There's some sort of weird race condition?
         # within Github that makes the sdist be missing files

.gitpod.yml

@@ -3,19 +3,20 @@
 # https://www.gitpod.io/docs/config-start-tasks/#configuring-the-terminal
 # -------------------------------------------------------------------------
 
-# assuming we use dockerhub: name of the docker user, docker image, tag, e.g. https://hub.docker.com/r/pandas/pandas-gitpod/tags
-image: pandas/pandas-gitpod:latest
+# images for gitpod pandas are in https://hub.docker.com/r/pandas/pandas-gitpod/tags
+# we're using the Dockerfile in the base of the repo
+image:
+  file: Dockerfile
 tasks:
   - name: Prepare development environment
     init: |
       mkdir -p .vscode
       cp gitpod/settings.json .vscode/settings.json
-      conda activate pandas-dev
-      git pull --unshallow  # need to force this else the prebuild fails
       git fetch --tags
-      python setup.py build_ext --inplace -j 4
-      echo "🛠 Completed rebuilding Pandas!! 🛠 "
+      python -m pip install -ve . --no-build-isolation --config-settings editable-verbose=true
       pre-commit install
+    command: |
+      python -m pip install -ve . --no-build-isolation --config-settings editable-verbose=true
       echo "✨ Pre-build complete! You can close this terminal ✨ "
 
 # --------------------------------------------------------
@@ -37,7 +38,7 @@ vscode:
     # avoid adding too many. they each open a pop-up window
 
 # --------------------------------------------------------
-# using prebuilds for the container
+# Using prebuilds for the container
 # With this configuration the prebuild will happen on push to main
 github:
   prebuilds:

.pre-commit-config.yaml

@@ -15,20 +15,13 @@ default_stages: [
 ci:
     autofix_prs: false
 repos:
--   repo: local
+-   repo: https://github.com/hauntsaninja/black-pre-commit-mirror
+    # black compiled with mypyc
+    rev: 23.3.0
     hooks:
-    # NOTE: we make `black` a local hook because if it's installed from
-    # PyPI (rather than from source) then it'll run twice as fast thanks to mypyc
-    -   id: black
-        name: black
-        description: "Black: The uncompromising Python code formatter"
-        entry: black
-        language: python
-        require_serial: true
-        types_or: [python, pyi]
-        additional_dependencies: [black==23.3.0]
--   repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: v0.0.270
+      - id: black
+-   repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.0.277
     hooks:
     -   id: ruff
         args: [--exit-non-zero-on-fix]
@@ -40,7 +33,7 @@ repos:
         pass_filenames: true
         require_serial: false
 -   repo: https://github.com/codespell-project/codespell
-    rev: v2.2.4
+    rev: v2.2.5
     hooks:
     -   id: codespell
         types_or: [python, rst, markdown, cython, c]
@@ -53,14 +46,22 @@ repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.4.0
     hooks:
+    -   id: check-ast
+    -   id: check-case-conflict
+    -   id: check-toml
+    -   id: check-xml
+    -   id: check-yaml
+        exclude: ^ci/meta.yaml$
     -   id: debug-statements
     -   id: end-of-file-fixer
         exclude: \.txt$
-        stages: [commit, merge-commit, push, prepare-commit-msg, commit-msg,
-                 post-checkout, post-commit, post-merge, post-rewrite]
+    -   id: mixed-line-ending
+        args: [--fix=auto]
+        exclude: ^pandas/tests/io/parser/data/utf16_ex.txt$
+    -   id: fix-byte-order-marker
+    -   id: fix-encoding-pragma
+        args: [--remove]
     -   id: trailing-whitespace
-        stages: [commit, merge-commit, push, prepare-commit-msg, commit-msg,
-                 post-checkout, post-commit, post-merge, post-rewrite]
 -   repo: https://github.com/cpplint/cpplint
     rev: 1.6.1
     hooks:
@@ -74,7 +75,7 @@ repos:
             --linelength=88,
             '--filter=-readability/casting,-runtime/int,-build/include_subdir,-readability/fn_size'
         ]
--   repo: https://github.com/pycqa/pylint
+-   repo: https://github.com/pylint-dev/pylint
     rev: v3.0.0a6
     hooks:
     -   id: pylint
@@ -93,23 +94,20 @@ repos:
             |^pandas/conftest\.py  # keep excluded
         args: [--disable=all, --enable=redefined-outer-name]
         stages: [manual]
-    -   id: pylint
-        alias: unspecified-encoding
-        name: Using open without explicitly specifying an encoding
-        args: [--disable=all, --enable=unspecified-encoding]
-        stages: [manual]
 -   repo: https://github.com/PyCQA/isort
     rev: 5.12.0
     hooks:
     -   id: isort
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v3.4.0
+    rev: v3.7.0
     hooks:
     -   id: pyupgrade
-        args: [--py38-plus]
+        args: [--py39-plus]
 -   repo: https://github.com/pre-commit/pygrep-hooks
     rev: v1.10.0
     hooks:
+      - id: python-check-blanket-noqa
+      - id: python-check-blanket-type-ignore
       - id: rst-backticks
       - id: rst-directive-colons
         types: [text]  # overwrite types: [rst]
@@ -132,7 +130,7 @@ repos:
         types: [python]
         stages: [manual]
         additional_dependencies: &pyright_dependencies
-        - [email protected].292
+        - [email protected].296
     -   id: pyright_reportGeneralTypeIssues
         # note: assumes python env is setup and activated
         name: pyright reportGeneralTypeIssues
@@ -166,34 +164,17 @@ repos:
         exclude: ^pandas/core/interchange/
         language: python
         types: [python]
-    -   id: no-os-remove
-        name: Check code for instances of os.remove
-        entry: os\.remove
-        language: pygrep
-        types: [python]
-        files: ^pandas/tests/
-        exclude: |
-            (?x)^
-            pandas/tests/io/pytables/test_store\.py$
     -   id: unwanted-patterns
         name: Unwanted patterns
         language: pygrep
         entry: |
             (?x)
-            # outdated annotation syntax, missing error codes
+            # outdated annotation syntax
             \#\ type:\ (?!ignore)
-            |\#\ type:\s?ignore(?!\[)
 
             # foo._class__ instead of type(foo)
             |\.__class__
 
-            # np.bool/np.object instead of np.bool_/np.object_
-            |np\.bool[^_8`]
-            |np\.object[^_8`]
-
-            # imports from collections.abc instead of `from collections import abc`
-            |from\ collections\.abc\ import
-
             # Numpy
             |from\ numpy\ import\ random
             |from\ numpy\.random\ import
@@ -212,16 +193,8 @@ repos:
 
             # builtin filter function
             |(?<!def)[\(\s]filter\(
-
-            # exec
-            |[^a-zA-Z0-9_]exec\(
         types_or: [python, cython, rst]
         exclude: ^doc/source/development/code_style\.rst  # contains examples of patterns to avoid
-    -   id: cython-casting
-        name: Check Cython casting is `<type>obj`, not `<type> obj`
-        language: pygrep
-        entry: '[a-zA-Z0-9*]> '
-        files: (\.pyx|\.pxi.in)$
     -   id: incorrect-backticks
         name: Check for backticks incorrectly rendering because of missing spaces
         language: pygrep
@@ -234,19 +207,6 @@ repos:
         entry: 'np\.random\.seed'
         files: ^asv_bench/benchmarks
         exclude: ^asv_bench/benchmarks/pandas_vb_common\.py
-    -   id: np-testing-array-equal
-        name: Check for usage of numpy testing or array_equal
-        language: pygrep
-        entry: '(numpy|np)(\.testing|\.array_equal)'
-        files: ^pandas/tests/
-        types: [python]
-    -   id: invalid-ea-testing
-        name: Check for invalid EA testing
-        language: pygrep
-        entry: 'tm\.assert_(series|frame)_equal'
-        files: ^pandas/tests/extension/base
-        types: [python]
-        exclude: ^pandas/tests/extension/base/base\.py
     -   id: unwanted-patterns-in-tests
         name: Unwanted patterns in tests
         language: pygrep
@@ -280,6 +240,9 @@ repos:
 
             # pytest.warns (use tm.assert_produces_warning instead)
             |pytest\.warns
+
+            # os.remove
+            |os\.remove
         files: ^pandas/tests/
         types_or: [python, cython, rst]
     -   id: unwanted-patterns-in-ea-tests

README.md

@@ -5,17 +5,13 @@
 -----------------
 
 # pandas: powerful Python data analysis toolkit
-[![PyPI Latest Release](https://img.shields.io/pypi/v/pandas.svg)](https://pypi.org/project/pandas/)
-[![Conda Latest Release](https://anaconda.org/conda-forge/pandas/badges/version.svg)](https://anaconda.org/anaconda/pandas/)
-[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.3509134.svg)](https://doi.org/10.5281/zenodo.3509134)
-[![Package Status](https://img.shields.io/pypi/status/pandas.svg)](https://pypi.org/project/pandas/)
-[![License](https://img.shields.io/pypi/l/pandas.svg)](https://github.com/pandas-dev/pandas/blob/main/LICENSE)
-[![Coverage](https://codecov.io/github/pandas-dev/pandas/coverage.svg?branch=main)](https://codecov.io/gh/pandas-dev/pandas)
-[![Downloads](https://static.pepy.tech/personalized-badge/pandas?period=month&units=international_system&left_color=black&right_color=orange&left_text=PyPI%20downloads%20per%20month)](https://pepy.tech/project/pandas)
-[![Slack](https://img.shields.io/badge/join_Slack-information-brightgreen.svg?logo=slack)](https://pandas.pydata.org/docs/dev/development/community.html?highlight=slack#community-slack)
-[![Powered by NumFOCUS](https://img.shields.io/badge/powered%20by-NumFOCUS-orange.svg?style=flat&colorA=E1523D&colorB=007D8A)](https://numfocus.org)
-[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
-[![Imports: isort](https://img.shields.io/badge/%20imports-isort-%231674b1?style=flat&labelColor=ef8336)](https://pycqa.github.io/isort/)
+
+| | |
+| --- | --- |
+| Testing | [![CI - Test](https://github.com/pandas-dev/pandas/actions/workflows/unit-tests.yml/badge.svg)](https://github.com/pandas-dev/pandas/actions/workflows/unit-tests.yml) [![Coverage](https://codecov.io/github/pandas-dev/pandas/coverage.svg?branch=main)](https://codecov.io/gh/pandas-dev/pandas) |
+| Package | [![PyPI Latest Release](https://img.shields.io/pypi/v/pandas.svg)](https://pypi.org/project/pandas/) [![PyPI Downloads](https://img.shields.io/pypi/dm/pandas.svg?label=PyPI%20downloads)](https://pypi.org/project/pandas/) [![Conda Latest Release](https://anaconda.org/conda-forge/pandas/badges/version.svg)](https://anaconda.org/conda-forge/pandas) [![Conda Downloads](https://img.shields.io/conda/dn/conda-forge/pandas.svg?label=Conda%20downloads)](https://anaconda.org/conda-forge/pandas) |
+| Meta | [![Powered by NumFOCUS](https://img.shields.io/badge/powered%20by-NumFOCUS-orange.svg?style=flat&colorA=E1523D&colorB=007D8A)](https://numfocus.org) [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.3509134.svg)](https://doi.org/10.5281/zenodo.3509134) [![License - BSD 3-Clause](https://img.shields.io/pypi/l/pandas.svg)](https://github.com/pandas-dev/pandas/blob/main/LICENSE) [![Slack](https://img.shields.io/badge/join_Slack-information-brightgreen.svg?logo=slack)](https://pandas.pydata.org/docs/dev/development/community.html?highlight=slack#community-slack) |
+
 
 ## What is it?
 
@@ -156,7 +152,7 @@ See the full instructions for [installing from source](https://pandas.pydata.org
 [BSD 3](LICENSE)
 
 ## Documentation
-The official documentation is hosted on PyData.org: https://pandas.pydata.org/pandas-docs/stable
+The official documentation is hosted on [PyData.org](https://pandas.pydata.org/pandas-docs/stable/).
 
 ## Background
 Work on ``pandas`` started at [AQR](https://www.aqr.com/) (a quantitative hedge fund) in 2008 and