Merge branch 'master' into Development

Author: Varun270

.circleci/config.yml

@@ -0,0 +1,21 @@
+version: 2.1
+
+jobs:
+  test-arm:
+    machine:
+      image: ubuntu-2004:202101-01
+    resource_class: arm.medium
+    environment:
+      ENV_FILE: ci/deps/circle-38-arm64.yaml
+      PYTEST_WORKERS: auto
+      PATTERN: "not slow and not network and not clipboard and not arm_slow"
+      PYTEST_TARGET: "pandas"
+    steps:
+      - checkout
+      - run: ci/setup_env.sh
+      - run: PATH=$HOME/miniconda3/envs/pandas-dev/bin:$HOME/miniconda3/condabin:$PATH ci/run_tests.sh
+
+workflows:
+  test:
+    jobs:
+      - test-arm

.github/actions/build_pandas/action.yml

@@ -13,5 +13,5 @@ runs:
     - name: Build Pandas
       run: |
         python setup.py build_ext -j 2
-        python -m pip install -e . --no-build-isolation --no-use-pep517
+        python -m pip install -e . --no-build-isolation --no-use-pep517 --no-index
       shell: bash -l {0}

.github/workflows/ci.yml

@@ -23,8 +23,9 @@ jobs:
         shell: bash -l {0}
 
     concurrency:
-      group: ${{ github.ref }}-checks
-      cancel-in-progress: ${{github.event_name == 'pull_request'}}
+      # https://github.community/t/concurrecy-not-work-for-push/183068/7
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-checks
+      cancel-in-progress: true
 
     steps:
     - name: Checkout
@@ -45,6 +46,15 @@ jobs:
         environment-file: ${{ env.ENV_FILE }}
         use-only-tar-bz2: true
 
+    - name: Install node.js (for pyright)
+      uses: actions/setup-node@v2
+      with:
+        node-version: "16"
+
+    - name: Install pyright
+      # note: keep version in sync with .pre-commit-config.yaml
+      run: npm install -g [email protected]
+
     - name: Build Pandas
       uses: ./.github/actions/build_pandas
 
@@ -93,7 +103,8 @@ jobs:
     runs-on: ubuntu-latest
 
     concurrency:
-      group: ${{ github.ref }}-web-docs
+      # https://github.community/t/concurrecy-not-work-for-push/183068/7
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-web-docs
       cancel-in-progress: true
 
     steps:
@@ -127,8 +138,11 @@ jobs:
         echo "${{ secrets.server_ip }} ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBE1Kkopomm7FHG5enATf7SgnpICZ4W2bw+Ho+afqin+w7sMcrsa0je7sbztFAV8YchDkiBKnWTG4cRT+KZgZCaY=" > ~/.ssh/known_hosts
       if: ${{github.event_name == 'push' && github.ref == 'refs/heads/master'}}
 
+    - name: Copy cheatsheets into site directory
+      run: cp doc/cheatsheet/Pandas_Cheat_Sheet* web/build/
+
     - name: Upload web
-      run: rsync -az --delete --exclude='pandas-docs' --exclude='docs' --exclude='Pandas_Cheat_Sheet*' web/build/ docs@${{ secrets.server_ip }}:/usr/share/nginx/pandas
+      run: rsync -az --delete --exclude='pandas-docs' --exclude='docs' web/build/ docs@${{ secrets.server_ip }}:/usr/share/nginx/pandas
       if: ${{github.event_name == 'push' && github.ref == 'refs/heads/master'}}
 
     - name: Upload dev docs
@@ -137,6 +151,7 @@ jobs:
 
     - name: Move docs into site directory
       run: mv doc/build/html web/build/docs
+
     - name: Save website as an artifact
       uses: actions/upload-artifact@v2
       with:
@@ -147,11 +162,20 @@ jobs:
   data_manager:
     name: Test experimental data manager
     runs-on: ubuntu-latest
+    services:
+      moto:
+        image: motoserver/moto
+        env:
+          AWS_ACCESS_KEY_ID: foobar_key
+          AWS_SECRET_ACCESS_KEY: foobar_secret
+        ports:
+          - 5000:5000
     strategy:
       matrix:
         pattern: ["not slow and not network and not clipboard", "slow"]
     concurrency:
-      group: ${{ github.ref }}-data_manager-${{ matrix.pattern }}
+      # https://github.community/t/concurrecy-not-work-for-push/183068/7
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-data_manager-${{ matrix.pattern }}
       cancel-in-progress: true
 
     steps:

.github/workflows/database.yml

@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - master
+      - 1.3.x
   pull_request:
     branches:
       - master
@@ -30,8 +31,9 @@ jobs:
       fail-fast: false
 
     concurrency:
-      group: ${{ github.ref }}-${{ matrix.ENV_FILE }}
-      cancel-in-progress: ${{github.event_name == 'pull_request'}}
+      # https://github.community/t/concurrecy-not-work-for-push/183068/7
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.ENV_FILE }}
+      cancel-in-progress: true
 
     services:
       mysql:
@@ -61,6 +63,14 @@ jobs:
         ports:
           - 5432:5432
 
+      moto:
+        image: motoserver/moto
+        env:
+          AWS_ACCESS_KEY_ID: foobar_key
+          AWS_SECRET_ACCESS_KEY: foobar_secret
+        ports:
+          - 5000:5000
+
     steps:
     - name: Checkout
       uses: actions/checkout@v2
@@ -79,7 +89,7 @@ jobs:
     - uses: conda-incubator/setup-miniconda@v2
       with:
         activate-environment: pandas-dev
-        channel-priority: flexible
+        channel-priority: strict
         environment-file: ${{ matrix.ENV_FILE }}
         use-only-tar-bz2: true
 

.github/workflows/posix.yml

@@ -27,12 +27,12 @@ jobs:
         settings: [
           [actions-38-minimum_versions.yaml, "not slow and not network and not clipboard", "", "", "", "", ""],
           [actions-38-locale_slow.yaml, "slow", "language-pack-it xsel", "it_IT.utf8", "it_IT.utf8", "", ""],
-          [actions-38.yaml, "not slow and not network and not clipboard", "", "", "", "", ""],
+          [actions-38.yaml, "not slow and not clipboard", "", "", "", "", ""],
           [actions-38-slow.yaml, "slow", "", "", "", "", ""],
           [actions-38-locale.yaml, "not slow and not network", "language-pack-zh-hans xsel", "zh_CN.utf8", "zh_CN.utf8", "", ""],
           [actions-39-slow.yaml, "slow", "", "", "", "", ""],
           [actions-39-numpydev.yaml, "not slow and not network", "xsel", "", "", "deprecate", "-W error"],
-          [actions-39.yaml, "not slow and not network and not clipboard", "", "", "", "", ""]
+          [actions-39.yaml, "not slow and not clipboard", "", "", "", "", ""]
         ]
       fail-fast: false
     env:
@@ -46,8 +46,17 @@ jobs:
       TEST_ARGS: ${{ matrix.settings[6] }}
       PYTEST_TARGET:  pandas
     concurrency:
-      group: ${{ github.ref }}-${{ matrix.settings[0] }}
-      cancel-in-progress: ${{github.event_name == 'pull_request'}}
+      # https://github.community/t/concurrecy-not-work-for-push/183068/7
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.settings[0] }}
+      cancel-in-progress: true
+    services:
+      moto:
+        image: motoserver/moto
+        env:
+          AWS_ACCESS_KEY_ID: foobar_key
+          AWS_SECRET_ACCESS_KEY: foobar_secret
+        ports:
+          - 5000:5000
 
     steps:
     - name: Checkout

.github/workflows/pre-commit.yml

@@ -11,9 +11,12 @@ jobs:
   pre-commit:
     runs-on: ubuntu-latest
     concurrency:
-      group: ${{ github.ref }}-pre-commit
-      cancel-in-progress: ${{github.event_name == 'pull_request'}}
+      # https://github.community/t/concurrecy-not-work-for-push/183068/7
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-pre-commit
+      cancel-in-progress: true
     steps:
     - uses: actions/checkout@v2
     - uses: actions/setup-python@v2
+      with:
+        python-version: '3.9.7'
     - uses: pre-commit/[email protected]

.github/workflows/python-dev.yml

@@ -17,17 +17,23 @@ env:
   PANDAS_CI: 1
   PATTERN: "not slow and not network and not clipboard"
   COVERAGE: true
-  PYTEST_TARGET:  pandas
+  PYTEST_TARGET: pandas
 
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macOS-latest, windows-latest]
+
     name: actions-310-dev
-    timeout-minutes: 60
+    timeout-minutes: 80
 
     concurrency:
-      group: ${{ github.ref }}-dev
-      cancel-in-progress: ${{github.event_name == 'pull_request'}}
+      #https://github.community/t/concurrecy-not-work-for-push/183068/7
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.os }}-${{ matrix.pytest_target }}-dev
+      cancel-in-progress: true
 
     steps:
     - uses: actions/checkout@v2
@@ -40,12 +46,12 @@ jobs:
         python-version: '3.10-dev'
 
     - name: Install dependencies
+      shell: bash
       run: |
         python -m pip install --upgrade pip setuptools wheel
         pip install -i https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy
-        pip install git+https://github.com/pytest-dev/pytest.git
         pip install git+https://github.com/nedbat/coveragepy.git
-        pip install cython python-dateutil pytz hypothesis pytest-xdist pytest-cov
+        pip install cython python-dateutil pytz hypothesis pytest>=6.2.5 pytest-xdist pytest-cov
         pip list
 
     - name: Build Pandas
@@ -58,10 +64,9 @@ jobs:
         python -c "import pandas; pandas.show_versions();"
 
     - name: Test with pytest
+      shell: bash
       run: |
         ci/run_tests.sh
-      # GH 41935
-      continue-on-error: true
 
     - name: Publish test results
       uses: actions/upload-artifact@master

.github/workflows/sdist.yml

@@ -23,10 +23,11 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9"]
+        python-version: ["3.8", "3.9", "3.10"]
     concurrency:
-      group: ${{github.ref}}-${{matrix.python-version}}-sdist
-      cancel-in-progress: ${{github.event_name == 'pull_request'}}
+      # https://github.community/t/concurrecy-not-work-for-push/183068/7
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{matrix.python-version}}-sdist
+      cancel-in-progress: true
 
     steps:
     - uses: actions/checkout@v2
@@ -53,13 +54,25 @@ jobs:
     - uses: conda-incubator/setup-miniconda@v2
       with:
         activate-environment: pandas-sdist
-        python-version: ${{ matrix.python-version }}
+        channels: conda-forge
+        python-version: '${{ matrix.python-version }}'
 
     - name: Install pandas from sdist
       run: |
-        conda list
+        pip list
         python -m pip install dist/*.gz
 
+    - name: Force oldest supported NumPy
+      run: |
+        case "${{matrix.python-version}}" in
+        3.8)
+          pip install numpy==1.18.5 ;;
+        3.9)
+          pip install numpy==1.19.3 ;;
+        3.10)
+          pip install numpy==1.21.2 ;;
+        esac
+
     - name: Import pandas
       run: |
         cd ..

.pre-commit-config.yaml

@@ -9,7 +9,7 @@ repos:
     -   id: absolufy-imports
         files: ^pandas/
 -   repo: https://github.com/python/black
-    rev: 21.7b0
+    rev: 21.11b1
     hooks:
     -   id: black
 -   repo: https://github.com/codespell-project/codespell
@@ -39,10 +39,11 @@ repos:
     rev: 3.9.2
     hooks:
     -   id: flake8
-        additional_dependencies:
-            - flake8-comprehensions==3.1.0
-            - flake8-bugbear==21.3.2
-            - pandas-dev-flaker==0.2.0
+        additional_dependencies: &flake8_dependencies
+        - flake8==3.9.2
+        - flake8-comprehensions==3.1.0
+        - flake8-bugbear==21.3.2
+        - pandas-dev-flaker==0.2.0
     -   id: flake8
         alias: flake8-cython
         name: flake8 (cython)
@@ -54,11 +55,11 @@ repos:
         types: [text]
         args: [--append-config=flake8/cython-template.cfg]
 -   repo: https://github.com/PyCQA/isort
-    rev: 5.9.3
+    rev: 5.10.1
     hooks:
     -   id: isort
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v2.23.3
+    rev: v2.29.1
     hooks:
     -   id: pyupgrade
         args: [--py38-plus]
@@ -73,14 +74,21 @@ repos:
         types: [text]  # overwrite types: [rst]
         types_or: [python, rst]
 -   repo: https://github.com/asottile/yesqa
-    rev: v1.2.3
+    rev: v1.3.0
     hooks:
     -   id: yesqa
-        additional_dependencies:
-            - flake8==3.9.2
-            - flake8-comprehensions==3.1.0
-            - flake8-bugbear==21.3.2
-            - pandas-dev-flaker==0.2.0
+        additional_dependencies: *flake8_dependencies
+-   repo: local
+    hooks:
+    -   id: pyright
+        name: pyright
+        entry: pyright
+        language: node
+        pass_filenames: false
+        types: [python]
+        stages: [manual]
+        # note: keep version in sync with .github/workflows/ci.yml
+        additional_dependencies: ['[email protected]']
 -   repo: local
     hooks:
     -   id: flake8-rst
@@ -124,6 +132,12 @@ repos:
         entry: 'np\.random\.seed'
         files: ^asv_bench/benchmarks
         exclude: ^asv_bench/benchmarks/pandas_vb_common\.py
+    -   id: np-testing-array-equal
+        name: Check for usage of numpy testing or array_equal
+        language: pygrep
+        entry: '(numpy|np)(\.testing|\.array_equal)'
+        files: ^pandas/tests/
+        types: [python]
     -   id: invalid-ea-testing
         name: Check for invalid EA testing
         language: pygrep

MANIFEST.in

@@ -33,6 +33,7 @@ global-exclude *.xlsb
 global-exclude *.xlsm
 global-exclude *.xlsx
 global-exclude *.xpt
+global-exclude *.cpt
 global-exclude *.xz
 global-exclude *.zip
 global-exclude *~

README.md

@@ -12,7 +12,7 @@
 [![License](https://img.shields.io/pypi/l/pandas.svg)](https://github.com/pandas-dev/pandas/blob/master/LICENSE)
 [![Azure Build Status](https://dev.azure.com/pandas-dev/pandas/_apis/build/status/pandas-dev.pandas?branch=master)](https://dev.azure.com/pandas-dev/pandas/_build/latest?definitionId=1&branch=master)
 [![Coverage](https://codecov.io/github/pandas-dev/pandas/coverage.svg?branch=master)](https://codecov.io/gh/pandas-dev/pandas)
-[![Downloads](https://anaconda.org/conda-forge/pandas/badges/downloads.svg)](https://pandas.pydata.org)
+[![Downloads](https://static.pepy.tech/personalized-badge/pandas?period=month&units=international_system&left_color=black&right_color=orange&left_text=PyPI%20downloads%20per%20month)](https://pepy.tech/project/pandas)
 [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/pydata/pandas)
 [![Powered by NumFOCUS](https://img.shields.io/badge/powered%20by-NumFOCUS-orange.svg?style=flat&colorA=E1523D&colorB=007D8A)](https://numfocus.org)
 [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)