Merge branch 'main' into patch-2

Author: jonashaag

.circleci/config.yml

@@ -13,7 +13,7 @@ jobs:
       PANDAS_CI: "1"
     steps:
       - checkout
-      - run: ci/setup_env.sh
+      - run: .circleci/setup_env.sh
       - run: PATH=$HOME/miniconda3/envs/pandas-dev/bin:$HOME/miniconda3/condabin:$PATH ci/run_tests.sh
 
 workflows:

ci/setup_env.sh renamed to .circleci/setup_env.sh

@@ -1,4 +1,5 @@
 - [ ] closes #xxxx (Replace xxxx with the Github issue number)
 - [ ] [Tests added and passed](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#writing-tests) if fixing a bug or adding a new feature
 - [ ] All [code checks passed](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#pre-commit).
+- [ ] Added [type annotations](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#type-hints) to new arguments/methods/functions.
 - [ ] Added an entry in the latest `doc/source/whatsnew/vX.X.X.rst` file if fixing a bug or adding a new feature.

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,37 @@
+name: Set up Conda environment
+inputs:
+  environment-file:
+    description: Conda environment file to use.
+    default: environment.yml
+  environment-name:
+    description: Name to use for the Conda environment
+    default: test
+  python-version:
+    description: Python version to install
+    required: false
+  pyarrow-version:
+    description: If set, overrides the PyArrow version in the Conda environment to the given string.
+    required: false
+runs:
+  using: composite
+  steps:
+    - name: Set Arrow version in ${{ inputs.environment-file }} to ${{ inputs.pyarrow-version }}
+      run: |
+        grep -q '  - pyarrow' ${{ inputs.environment-file }}
+        sed -i"" -e "s/  - pyarrow/  - pyarrow=${{ inputs.pyarrow-version }}/" ${{ inputs.environment-file }}
+        cat ${{ inputs.environment-file }}
+      shell: bash
+      if: ${{ inputs.pyarrow-version }}
+
+    - name: Install ${{ inputs.environment-file }}
+      uses: conda-incubator/[email protected]
+      with:
+        environment-file: ${{ inputs.environment-file }}
+        activate-environment: ${{ inputs.environment-name }}
+        python-version: ${{ inputs.python-version }}
+        channel-priority: ${{ runner.os == 'macOS' && 'flexible' || 'strict' }}
+        channels: conda-forge
+        mamba-version: "0.24"
+        use-mamba: true
+        use-only-tar-bz2: true
+        condarc-file: ci/condarc.yml

.github/actions/setup-conda/action.yml

@@ -23,9 +23,15 @@ jobs:
 
       - name: Run 32-bit manylinux2014 Docker Build / Tests
         run: |
+          # Without this (line 34), versioneer will not be able to determine the pandas version.
+          # This is because of a security update to git that blocks it from reading the config folder if
+          # it is not owned by the current user. We hit this since the "mounted" folder is not hit by the
+          # Docker container.
+          # xref https://github.com/pypa/manylinux/issues/1309
           docker pull quay.io/pypa/manylinux2014_i686
           docker run --platform linux/386 -v $(pwd):/pandas quay.io/pypa/manylinux2014_i686 \
           /bin/bash -xc "cd pandas && \
+          git config --global --add safe.directory /pandas && \
           /opt/python/cp38-cp38/bin/python -m venv ~/virtualenvs/pandas-dev && \
           . ~/virtualenvs/pandas-dev/bin/activate && \
           python -m pip install --no-deps -U pip wheel 'setuptools<60.0.0' && \

.github/actions/setup/action.yml

@@ -41,12 +41,8 @@ jobs:
 
         # Although asv sets up its own env, deps are still needed
         # during discovery process
-      - uses: conda-incubator/[email protected]
-        with:
-          activate-environment: pandas-dev
-          channel-priority: strict
-          environment-file: ${{ env.ENV_FILE }}
-          use-only-tar-bz2: true
+      - name: Set up Conda
+        uses: ./.github/actions/setup-conda
 
       - name: Run benchmarks
         id: bench

.github/workflows/32-bit-linux.yml

@@ -58,14 +58,8 @@ jobs:
         path: ~/conda_pkgs_dir
         key: ${{ runner.os }}-conda-${{ hashFiles('${{ env.ENV_FILE }}') }}
 
-    - uses: conda-incubator/[email protected]
-      with:
-        mamba-version: "*"
-        channels: conda-forge
-        activate-environment: pandas-dev
-        channel-priority: strict
-        environment-file: ${{ env.ENV_FILE }}
-        use-only-tar-bz2: true
+    - name: Set up Conda
+      uses: ./.github/actions/setup-conda
 
     - name: Build Pandas
       id: build
@@ -127,14 +121,8 @@ jobs:
         path: ~/conda_pkgs_dir
         key: ${{ runner.os }}-conda-${{ hashFiles('${{ env.ENV_FILE }}') }}
 
-    - uses: conda-incubator/[email protected]
-      with:
-        mamba-version: "*"
-        channels: conda-forge
-        activate-environment: pandas-dev
-        channel-priority: strict
-        environment-file: ${{ env.ENV_FILE }}
-        use-only-tar-bz2: true
+    - name: Set up Conda
+      uses: ./.github/actions/setup-conda
 
     - name: Build Pandas
       id: build

.github/workflows/asv-bot.yml

@@ -24,43 +24,46 @@ jobs:
       group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-web-docs
       cancel-in-progress: true
 
+    defaults:
+      run:
+        shell: bash -el {0}
+
     steps:
     - name: Checkout
       uses: actions/checkout@v3
       with:
         fetch-depth: 0
 
-    - name: Set up pandas
-      uses: ./.github/actions/setup
+    - name: Set up Conda
+      uses: ./.github/actions/setup-conda
+
+    - name: Build Pandas
+      uses: ./.github/actions/build_pandas
 
     - name: Build website
-      run: |
-        source activate pandas-dev
-        python web/pandas_web.py web/pandas --target-path=web/build
+      run: python web/pandas_web.py web/pandas --target-path=web/build
 
     - name: Build documentation
-      run: |
-        source activate pandas-dev
-        doc/make.py --warnings-are-errors
+      run: doc/make.py --warnings-are-errors
 
     - name: Install ssh key
       run: |
         mkdir -m 700 -p ~/.ssh
         echo "${{ secrets.server_ssh_key }}" > ~/.ssh/id_rsa
         chmod 600 ~/.ssh/id_rsa
         echo "${{ secrets.server_ip }} ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBE1Kkopomm7FHG5enATf7SgnpICZ4W2bw+Ho+afqin+w7sMcrsa0je7sbztFAV8YchDkiBKnWTG4cRT+KZgZCaY=" > ~/.ssh/known_hosts
-      if: ${{github.event_name == 'push' && github.ref == 'refs/heads/main'}}
+      if: github.event_name == 'push' && github.ref == 'refs/heads/main'
 
     - name: Copy cheatsheets into site directory
       run: cp doc/cheatsheet/Pandas_Cheat_Sheet* web/build/
 
     - name: Upload web
       run: rsync -az --delete --exclude='pandas-docs' --exclude='docs' web/build/ docs@${{ secrets.server_ip }}:/usr/share/nginx/pandas
-      if: ${{github.event_name == 'push' && github.ref == 'refs/heads/main'}}
+      if: github.event_name == 'push' && github.ref == 'refs/heads/main'
 
     - name: Upload dev docs
       run: rsync -az --delete doc/build/html/ docs@${{ secrets.server_ip }}:/usr/share/nginx/pandas/pandas-docs/dev
-      if: ${{github.event_name == 'push' && github.ref == 'refs/heads/main'}}
+      if: github.event_name == 'push' && github.ref == 'refs/heads/main'
 
     - name: Move docs into site directory
       run: mv doc/build/html web/build/docs

.github/workflows/code-checks.yml

@@ -43,32 +43,18 @@ jobs:
       with:
         fetch-depth: 0
 
-    - name: Install Dependencies
-      uses: conda-incubator/setup-[email protected]
+    - name: Set up Conda
+      uses: ./.github/actions/setup-conda
       with:
-        mamba-version: "*"
-        channels: conda-forge
-        activate-environment: pandas-dev
-        channel-priority: ${{ matrix.os == 'macos-latest' && 'flexible' || 'strict' }}
         environment-file: ci/deps/${{ matrix.env_file }}
-        use-only-tar-bz2: true
-
-    # ImportError: 2): Library not loaded: @rpath/libssl.1.1.dylib
-    # Referenced from: /Users/runner/miniconda3/envs/pandas-dev/lib/libthrift.0.13.0.dylib
-    # Reason: image not found
-    - name: Upgrade pyarrow on MacOS
-      run: conda install -n pandas-dev -c conda-forge --no-update-deps pyarrow=6
-      if: ${{ matrix.os == 'macos-latest' }}
+        pyarrow-version: ${{ matrix.os == 'macos-latest' && '6' || '' }}
 
     - name: Build Pandas
       uses: ./.github/actions/build_pandas
 
     - name: Test
       run: ci/run_tests.sh
 
-    - name: Build Version
-      run: conda list
-
     - name: Publish test results
       uses: actions/upload-artifact@v3
       with:

.github/workflows/docbuild-and-upload.yml

@@ -147,18 +147,11 @@ jobs:
       # xsel for clipboard tests
       run: sudo apt-get update && sudo apt-get install -y libc6-dev-i386 xsel ${{ env.EXTRA_APT }}
 
-    - uses: conda-incubator/[email protected]
+    - name: Set up Conda
+      uses: ./.github/actions/setup-conda
       with:
-        mamba-version: "*"
-        channels: conda-forge
-        activate-environment: pandas-dev
-        channel-priority: flexible
         environment-file: ${{ env.ENV_FILE }}
-        use-only-tar-bz2: true
-
-    - name: Upgrade Arrow version
-      run: conda install -n pandas-dev -c conda-forge --no-update-deps pyarrow=${{ matrix.pyarrow_version }}
-      if: ${{ matrix.pyarrow_version }}
+        pyarrow-version: ${{ matrix.pyarrow_version }}
 
     - name: Build Pandas
       uses: ./.github/actions/build_pandas

.github/workflows/macos-windows.yml

@@ -59,11 +59,12 @@ jobs:
         name: ${{matrix.python-version}}-sdist.gz
         path: dist/*.gz
 
-    - uses: conda-incubator/[email protected]
+    - name: Set up Conda
+      uses: ./.github/actions/setup-conda
       with:
-        activate-environment: pandas-sdist
-        channels: conda-forge
-        python-version: '${{ matrix.python-version }}'
+        environment-file: ""
+        environment-name: pandas-sdist
+        python-version: ${{ matrix.python-version }}
 
     - name: Install pandas from sdist
       run: |
@@ -74,9 +75,9 @@ jobs:
       run: |
         case "${{matrix.python-version}}" in
         3.8)
-          pip install numpy==1.18.5 ;;
+          pip install numpy==1.19.5 ;;
         3.9)
-          pip install numpy==1.19.3 ;;
+          pip install numpy==1.19.5 ;;
         3.10)
           pip install numpy==1.21.2 ;;
         esac

.github/workflows/posix.yml

@@ -11,7 +11,7 @@ repos:
     -   id: absolufy-imports
         files: ^pandas/
 -   repo: https://github.com/jendrikseipp/vulture
-    rev: 'v2.3'
+    rev: 'v2.4'
     hooks:
       - id: vulture
         entry: python scripts/run_vulture.py
@@ -26,7 +26,6 @@ repos:
     hooks:
     -   id: codespell
         types_or: [python, rst, markdown]
-        files: ^(pandas|doc)/
 -   repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.2.0
     hooks:
@@ -60,7 +59,7 @@ repos:
     hooks:
     -   id: isort
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v2.32.0
+    rev: v2.32.1
     hooks:
     -   id: pyupgrade
         args: [--py38-plus]
@@ -75,7 +74,7 @@ repos:
         types: [text]  # overwrite types: [rst]
         types_or: [python, rst]
 -   repo: https://github.com/sphinx-contrib/sphinx-lint
-    rev: v0.4.1
+    rev: v0.6
     hooks:
     - id: sphinx-lint
 -   repo: https://github.com/asottile/yesqa
@@ -93,7 +92,19 @@ repos:
         pass_filenames: false
         types: [python]
         stages: [manual]
-        additional_dependencies: ['[email protected]']
+        additional_dependencies: &pyright_dependencies
+        - [email protected]
+-   repo: local
+    hooks:
+    -   id: pyright_reportGeneralTypeIssues
+        name: pyright reportGeneralTypeIssues
+        entry: pyright --skipunannotated -p pyright_reportGeneralTypeIssues.json
+        # note: assumes python env is setup and activated
+        language: node
+        pass_filenames: false
+        types: [python]
+        stages: [manual]
+        additional_dependencies: *pyright_dependencies
 -   repo: local
     hooks:
     -   id: mypy

.github/workflows/sdist.yml

@@ -42,7 +42,7 @@
     // followed by the pip installed packages).
     "matrix": {
         "numpy": [],
-        "Cython": ["0.29.24"],
+        "Cython": ["0.29.30"],
         "matplotlib": [],
         "sqlalchemy": [],
         "scipy": [],

.pre-commit-config.yaml

@@ -37,7 +37,7 @@ def setup(self):
         self.dict_list = frame.to_dict(orient="records")
         self.data2 = {i: {j: float(j) for j in range(100)} for i in range(2000)}
 
-        # arrays which we wont consolidate
+        # arrays which we won't consolidate
         self.dict_of_categoricals = {i: Categorical(np.arange(N)) for i in range(K)}
 
     def time_list_of_dict(self):
@@ -60,7 +60,7 @@ def time_nested_dict_int64(self):
         DataFrame(self.data2)
 
     def time_dict_of_categoricals(self):
-        # dict of arrays that we wont consolidate
+        # dict of arrays that we won't consolidate
         DataFrame(self.dict_of_categoricals)
 
 

asv_bench/asv.conf.json

@@ -288,6 +288,26 @@ def time_values_mixed_wide(self):
         self.df_mixed_wide.values
 
 
+class ToRecords:
+    def setup(self):
+        N = 100_000
+        data = np.random.randn(N, 2)
+        mi = MultiIndex.from_arrays(
+            [
+                np.arange(N),
+                date_range("1970-01-01", periods=N, freq="ms"),
+            ]
+        )
+        self.df = DataFrame(data)
+        self.df_mi = DataFrame(data, index=mi)
+
+    def time_to_records(self):
+        self.df.to_records(index=True)
+
+    def time_to_records_multiindex(self):
+        self.df_mi.to_records(index=True)
+
+
 class Repr:
     def setup(self):
         nrows = 10000