Merge branch 'main' of https://github.com/pandas-dev/pandas into depr_core_groupby

Author: rhshadrach

.circleci/config.yml

@@ -49,7 +49,12 @@ jobs:
           no_output_timeout: 30m # Sometimes the tests won't generate any output, make sure the job doesn't get killed by that
           command: |
             pip3 install cibuildwheel==2.15.0
+            # When this is a nightly wheel build, allow picking up NumPy 2.0 dev wheels:
+            if [[ "$IS_SCHEDULE_DISPATCH" == "true" || "$IS_PUSH" != 'true' ]]; then
+                export CIBW_ENVIRONMENT="PIP_EXTRA_INDEX_URL=https://pypi.anaconda.org/scientific-python-nightly-wheels/simple"
+            fi
             cibuildwheel --prerelease-pythons --output-dir wheelhouse
+
           environment:
             CIBW_BUILD: << parameters.cibw-build >>
 

.github/workflows/unit-tests.yml

@@ -69,6 +69,10 @@ jobs:
             env_file: actions-311.yaml
             pattern: "not slow and not network and not single_cpu"
             pandas_copy_on_write: "1"
+          - name: "Copy-on-Write 3.11 (warnings)"
+            env_file: actions-311.yaml
+            pattern: "not slow and not network and not single_cpu"
+            pandas_copy_on_write: "warn"
           - name: "Pypy"
             env_file: actions-pypy-39.yaml
             pattern: "not slow and not network and not single_cpu"
@@ -94,7 +98,7 @@ jobs:
       PYTEST_TARGET: ${{ matrix.pytest_target || 'pandas' }}
     concurrency:
       # https://github.community/t/concurrecy-not-work-for-push/183068/7
-      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.env_file }}-${{ matrix.pattern }}-${{ matrix.extra_apt || '' }}
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.env_file }}-${{ matrix.pattern }}-${{ matrix.extra_apt || '' }}-${{ matrix.pandas_copy_on_write || '' }}
       cancel-in-progress: true
 
     services:
@@ -236,7 +240,7 @@ jobs:
           . ~/virtualenvs/pandas-dev/bin/activate
           python -m pip install --no-cache-dir -U pip wheel setuptools meson[ninja]==1.2.1 meson-python==0.13.1
           python -m pip install numpy --config-settings=setup-args="-Dallow-noblas=true"
-          python -m pip install --no-cache-dir versioneer[toml] "cython<3.0.3" python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-asyncio>=0.17 hypothesis>=6.46.1
+          python -m pip install --no-cache-dir versioneer[toml] cython python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-asyncio>=0.17 hypothesis>=6.46.1
           python -m pip install --no-cache-dir --no-build-isolation -e .
           python -m pip list --no-cache-dir
           export PANDAS_CI=1
@@ -274,7 +278,7 @@ jobs:
           /opt/python/cp311-cp311/bin/python -m venv ~/virtualenvs/pandas-dev
           . ~/virtualenvs/pandas-dev/bin/activate
           python -m pip install --no-cache-dir -U pip wheel setuptools meson-python==0.13.1 meson[ninja]==1.2.1
-          python -m pip install --no-cache-dir versioneer[toml] "cython<3.0.3" numpy python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-asyncio>=0.17 hypothesis>=6.46.1
+          python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-asyncio>=0.17 hypothesis>=6.46.1
           python -m pip install --no-cache-dir --no-build-isolation -e .
           python -m pip list --no-cache-dir
 
@@ -347,7 +351,7 @@ jobs:
           python -m pip install --upgrade pip setuptools wheel meson[ninja]==1.2.1 meson-python==0.13.1
           python -m pip install --pre --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy
           python -m pip install versioneer[toml]
-          python -m pip install python-dateutil pytz tzdata "cython<3.0.3" hypothesis>=6.46.1 pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-cov pytest-asyncio>=0.17
+          python -m pip install python-dateutil pytz tzdata cython hypothesis>=6.46.1 pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-cov pytest-asyncio>=0.17
           python -m pip install -ve . --no-build-isolation --no-index --no-deps
           python -m pip list
 

.github/workflows/wheels.yml

@@ -137,14 +137,27 @@ jobs:
         shell: bash -el {0}
         run: echo "sdist_name=$(cd ./dist && ls -d */)" >> "$GITHUB_ENV"
 
-      - name: Build wheels
+      - name: Build normal wheels
+        if: ${{ (env.IS_SCHEDULE_DISPATCH != 'true' || env.IS_PUSH == 'true') }}
         uses: pypa/[email protected]
         with:
          package-dir: ./dist/${{ matrix.buildplat[1] == 'macosx_*' && env.sdist_name || needs.build_sdist.outputs.sdist_file }}
         env:
           CIBW_PRERELEASE_PYTHONS: True
           CIBW_BUILD: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }}
 
+      - name: Build nightly wheels (with NumPy pre-release)
+        if: ${{ (env.IS_SCHEDULE_DISPATCH == 'true' && env.IS_PUSH != 'true') }}
+        uses: pypa/[email protected]
+        with:
+         package-dir: ./dist/${{ matrix.buildplat[1] == 'macosx_*' && env.sdist_name || needs.build_sdist.outputs.sdist_file }}
+        env:
+          # The nightly wheels should be build witht he NumPy 2.0 pre-releases
+          # which requires the additional URL.
+          CIBW_ENVIRONMENT: PIP_EXTRA_INDEX_URL=https://pypi.anaconda.org/scientific-python-nightly-wheels/simple
+          CIBW_PRERELEASE_PYTHONS: True
+          CIBW_BUILD: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }}
+
       - name: Set up Python
         uses: mamba-org/setup-micromamba@v1
         with:

.pre-commit-config.yaml

@@ -70,19 +70,6 @@ repos:
     -   id: fix-encoding-pragma
         args: [--remove]
     -   id: trailing-whitespace
--   repo: https://github.com/cpplint/cpplint
-    rev: 1.6.1
-    hooks:
-    -   id: cpplint
-        exclude: ^pandas/_libs/include/pandas/vendored/klib
-        args: [
-            --quiet,
-            '--extensions=c,h',
-            '--headers=h',
-            --recursive,
-            --linelength=88,
-            '--filter=-readability/casting,-runtime/int,-build/include_subdir,-readability/fn_size'
-        ]
 -   repo: https://github.com/pylint-dev/pylint
     rev: v3.0.0b0
     hooks:
@@ -127,6 +114,13 @@ repos:
     rev: v0.6.8
     hooks:
     - id: sphinx-lint
+-   repo: https://github.com/pre-commit/mirrors-clang-format
+    rev: ea59a72
+    hooks:
+    - id: clang-format
+      files: ^pandas/_libs/src|^pandas/_libs/include
+      args: [-i]
+      types_or: [c, c++]
 -   repo: local
     hooks:
     -   id: pyright

asv_bench/benchmarks/groupby.py

@@ -73,6 +73,8 @@
     "ffill",
     "first",
     "head",
+    "idxmax",
+    "idxmin",
     "last",
     "median",
     "nunique",
@@ -588,6 +590,8 @@ class GroupByCythonAgg:
             "prod",
             "min",
             "max",
+            "idxmin",
+            "idxmax",
             "mean",
             "median",
             "var",

ci/deps/actions-310.yaml

@@ -6,7 +6,7 @@ dependencies:
 
   # build dependencies
   - versioneer[toml]
-  - cython>=0.29.33, <3.0.3
+  - cython>=0.29.33
   - meson[ninja]=1.2.1
   - meson-python=0.13.1
 
@@ -24,39 +24,41 @@ dependencies:
   - pytz
 
   # optional dependencies
-  - beautifulsoup4>=4.11.1
-  - blosc>=1.21.0
-  - bottleneck>=1.3.4
-  - fastparquet>=0.8.1
-  - fsspec>=2022.05.0
+  - beautifulsoup4>=4.11.2
+  - blosc>=1.21.3
+  - bottleneck>=1.3.6
+  - fastparquet>=2022.12.0
+  - fsspec>=2022.11.0
   - html5lib>=1.1
   - hypothesis>=6.46.1
-  - gcsfs>=2022.05.0
+  - gcsfs>=2022.11.0
   - jinja2>=3.1.2
-  - lxml>=4.8.0
-  - matplotlib>=3.6.1, <3.8
-  - numba>=0.55.2
-  - numexpr>=2.8.0
+  - lxml>=4.9.2
+  - matplotlib>=3.6.3, <3.8
+  - numba>=0.56.4
+  - numexpr>=2.8.4
   - odfpy>=1.4.1
-  - qtpy>=2.2.0
-  - openpyxl>=3.0.10
-  - pandas-gbq>=0.17.5
-  - psycopg2>=2.9.3
-  - pyarrow>=7.0.0
+  - qtpy>=2.3.0
+  - openpyxl>=3.1.0
+  # Doesn't install well with pyarrow
+  # https://github.com/pandas-dev/pandas/issues/55525
+  # - pandas-gbq>=0.19.0
+  - psycopg2>=2.9.6
+  - pyarrow>=10.0.1
   - pymysql>=1.0.2
-  - pyreadstat>=1.1.5
-  - pytables>=3.7.0
+  - pyreadstat>=1.2.0
+  - pytables>=3.8.0
   - python-calamine>=0.1.6
-  - pyxlsb>=1.0.9
-  - s3fs>=2022.05.0
-  - scipy>=1.8.1
-  - sqlalchemy>=1.4.36
-  - tabulate>=0.8.10
-  - xarray>=2022.03.0
+  - pyxlsb>=1.0.10
+  - s3fs>=2022.11.0
+  - scipy>=1.10.0
+  - sqlalchemy>=2.0.0
+  - tabulate>=0.9.0
+  - xarray>=2022.12.0
   - xlrd>=2.0.1
-  - xlsxwriter>=3.0.3
-  - zstandard>=0.17.0
+  - xlsxwriter>=3.0.5
+  - zstandard>=0.19.0
 
   - pip:
-    - pyqt5>=5.15.6
-    - tzdata>=2022.1
+    - pyqt5>=5.15.8
+    - tzdata>=2022.7

ci/deps/actions-311-downstream_compat.yaml

@@ -7,7 +7,7 @@ dependencies:
 
   # build dependencies
   - versioneer[toml]
-  - cython>=0.29.33, <3.0.3
+  - cython>=0.29.33
   - meson[ninja]=1.2.1
   - meson-python=0.13.1
 
@@ -25,38 +25,40 @@ dependencies:
   - pytz
 
   # optional dependencies
-  - beautifulsoup4>=4.11.1
-  - blosc>=1.21.0
-  - bottleneck>=1.3.4
-  - fastparquet>=0.8.1
-  - fsspec>=2022.05.0
+  - beautifulsoup4>=4.11.2
+  - blosc>=1.21.3
+  - bottleneck>=1.3.6
+  - fastparquet>=2022.12.0
+  - fsspec>=2022.11.0
   - html5lib>=1.1
   - hypothesis>=6.46.1
-  - gcsfs>=2022.05.0
+  - gcsfs>=2022.11.0
   - jinja2>=3.1.2
-  - lxml>=4.8.0
-  - matplotlib>=3.6.1, <3.8
-  - numba>=0.55.2
-  - numexpr>=2.8.0
+  - lxml>=4.9.2
+  - matplotlib>=3.6.3, <3.8
+  - numba>=0.56.4
+  - numexpr>=2.8.4
   - odfpy>=1.4.1
-  - qtpy>=2.2.0
-  - openpyxl>=3.0.10
-  - pandas-gbq>=0.17.5
-  - psycopg2>=2.9.3
-  - pyarrow>=7.0.0
+  - qtpy>=2.3.0
+  - openpyxl>=3.1.0
+  # Doesn't install well with pyarrow
+  # https://github.com/pandas-dev/pandas/issues/55525
+  # - pandas-gbq>=0.19.0
+  - psycopg2>=2.9.6
+  - pyarrow>=10.0.1
   - pymysql>=1.0.2
-  - pyreadstat>=1.1.5
-  - pytables>=3.7.0
+  - pyreadstat>=1.2.0
+  - pytables>=3.8.0
   - python-calamine>=0.1.6
-  - pyxlsb>=1.0.9
-  - s3fs>=2022.05.0
-  - scipy>=1.8.1
-  - sqlalchemy>=1.4.36
-  - tabulate>=0.8.10
-  - xarray>=2022.03.0
+  - pyxlsb>=1.0.10
+  - s3fs>=2022.11.0
+  - scipy>=1.10.0
+  - sqlalchemy>=2.0.0
+  - tabulate>=0.9.0
+  - xarray>=2022.12.0
   - xlrd>=2.0.1
-  - xlsxwriter>=3.0.3
-  - zstandard>=0.17.0
+  - xlsxwriter>=3.0.5
+  - zstandard>=0.19.0
 
   # downstream packages
   - botocore
@@ -73,5 +75,5 @@ dependencies:
   - py
   - pip:
     - dataframe-api-compat>=0.1.7
-    - pyqt5>=5.15.6
-    - tzdata>=2022.1
+    - pyqt5>=5.15.8
+    - tzdata>=2022.7

ci/deps/actions-311-numpydev.yaml

@@ -8,7 +8,7 @@ dependencies:
   - versioneer[toml]
   - meson[ninja]=1.2.1
   - meson-python=0.13.1
-  - cython>=0.29.33, <3.0.3
+  - cython>=0.29.33
 
   # test dependencies
   - pytest>=7.3.2
@@ -29,5 +29,4 @@ dependencies:
     - "--extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple"
     - "--pre"
     - "numpy"
-    - "scipy"
-    - "tzdata>=2022.1"
+    - "tzdata>=2022.7"

ci/deps/actions-311-pyarrownightly.yaml

@@ -7,7 +7,7 @@ dependencies:
   # build dependencies
   - versioneer[toml]
   - meson[ninja]=1.2.1
-  - cython>=0.29.33, <3.0.3
+  - cython>=0.29.33
   - meson-python=0.13.1
 
   # test dependencies
@@ -24,7 +24,7 @@ dependencies:
   - pip
 
   - pip:
-    - "tzdata>=2022.1"
+    - "tzdata>=2022.7"
     - "--extra-index-url https://pypi.fury.io/arrow-nightlies/"
     - "--prefer-binary"
     - "--pre"