Merge branch 'main' of https://github.com/pandas-dev/pandas into numeric_only_window

Author: rhshadrach

.circleci/config.yml

@@ -13,7 +13,7 @@ jobs:
       PANDAS_CI: "1"
     steps:
       - checkout
-      - run: ci/setup_env.sh
+      - run: .circleci/setup_env.sh
       - run: PATH=$HOME/miniconda3/envs/pandas-dev/bin:$HOME/miniconda3/condabin:$PATH ci/run_tests.sh
 
 workflows:

ci/setup_env.sh renamed to .circleci/setup_env.sh

@@ -1,4 +1,5 @@
 - [ ] closes #xxxx (Replace xxxx with the Github issue number)
 - [ ] [Tests added and passed](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#writing-tests) if fixing a bug or adding a new feature
 - [ ] All [code checks passed](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#pre-commit).
+- [ ] Added [type annotations](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#type-hints) to new arguments/methods/functions.
 - [ ] Added an entry in the latest `doc/source/whatsnew/vX.X.X.rst` file if fixing a bug or adding a new feature.

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,27 @@
+name: Set up Conda environment
+inputs:
+  environment-file:
+    description: Conda environment file to use.
+    default: environment.yml
+  pyarrow-version:
+    description: If set, overrides the PyArrow version in the Conda environment to the given string.
+    required: false
+runs:
+  using: composite
+  steps:
+    - name: Set Arrow version in ${{ inputs.environment-file }} to ${{ inputs.pyarrow-version }}
+      run: |
+        grep -q '  - pyarrow' ${{ inputs.environment-file }}
+        sed -i"" -e "s/  - pyarrow/  - pyarrow=${{ inputs.pyarrow-version }}/" ${{ inputs.environment-file }}
+        cat ${{ inputs.environment-file }}
+      shell: bash
+      if: ${{ inputs.pyarrow-version }}
+
+    - name: Install ${{ inputs.environment-file }}
+      uses: conda-incubator/setup-miniconda@v2
+      with:
+        environment-file: ${{ inputs.environment-file }}
+        channel-priority: ${{ runner.os == 'macOS' && 'flexible' || 'strict' }}
+        channels: conda-forge
+        mamba-version: "0.23"
+        use-mamba: true

.github/actions/setup-conda/action.yml

@@ -23,9 +23,15 @@ jobs:
 
       - name: Run 32-bit manylinux2014 Docker Build / Tests
         run: |
+          # Without this (line 34), versioneer will not be able to determine the pandas version.
+          # This is because of a security update to git that blocks it from reading the config folder if
+          # it is not owned by the current user. We hit this since the "mounted" folder is not hit by the
+          # Docker container.
+          # xref https://github.com/pypa/manylinux/issues/1309
           docker pull quay.io/pypa/manylinux2014_i686
           docker run --platform linux/386 -v $(pwd):/pandas quay.io/pypa/manylinux2014_i686 \
           /bin/bash -xc "cd pandas && \
+          git config --global --add safe.directory /pandas && \
           /opt/python/cp38-cp38/bin/python -m venv ~/virtualenvs/pandas-dev && \
           . ~/virtualenvs/pandas-dev/bin/activate && \
           python -m pip install --no-deps -U pip wheel 'setuptools<60.0.0' && \

.github/actions/setup/action.yml

@@ -24,43 +24,46 @@ jobs:
       group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-web-docs
       cancel-in-progress: true
 
+    defaults:
+      run:
+        shell: bash -el {0}
+
     steps:
     - name: Checkout
       uses: actions/checkout@v3
       with:
         fetch-depth: 0
 
-    - name: Set up pandas
-      uses: ./.github/actions/setup
+    - name: Set up Conda
+      uses: ./.github/actions/setup-conda
+
+    - name: Build Pandas
+      uses: ./.github/actions/build_pandas
 
     - name: Build website
-      run: |
-        source activate pandas-dev
-        python web/pandas_web.py web/pandas --target-path=web/build
+      run: python web/pandas_web.py web/pandas --target-path=web/build
 
     - name: Build documentation
-      run: |
-        source activate pandas-dev
-        doc/make.py --warnings-are-errors
+      run: doc/make.py --warnings-are-errors
 
     - name: Install ssh key
       run: |
         mkdir -m 700 -p ~/.ssh
         echo "${{ secrets.server_ssh_key }}" > ~/.ssh/id_rsa
         chmod 600 ~/.ssh/id_rsa
         echo "${{ secrets.server_ip }} ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBE1Kkopomm7FHG5enATf7SgnpICZ4W2bw+Ho+afqin+w7sMcrsa0je7sbztFAV8YchDkiBKnWTG4cRT+KZgZCaY=" > ~/.ssh/known_hosts
-      if: ${{github.event_name == 'push' && github.ref == 'refs/heads/main'}}
+      if: github.event_name == 'push' && github.ref == 'refs/heads/main'
 
     - name: Copy cheatsheets into site directory
       run: cp doc/cheatsheet/Pandas_Cheat_Sheet* web/build/
 
     - name: Upload web
       run: rsync -az --delete --exclude='pandas-docs' --exclude='docs' web/build/ docs@${{ secrets.server_ip }}:/usr/share/nginx/pandas
-      if: ${{github.event_name == 'push' && github.ref == 'refs/heads/main'}}
+      if: github.event_name == 'push' && github.ref == 'refs/heads/main'
 
     - name: Upload dev docs
       run: rsync -az --delete doc/build/html/ docs@${{ secrets.server_ip }}:/usr/share/nginx/pandas/pandas-docs/dev
-      if: ${{github.event_name == 'push' && github.ref == 'refs/heads/main'}}
+      if: github.event_name == 'push' && github.ref == 'refs/heads/main'
 
     - name: Move docs into site directory
       run: mv doc/build/html web/build/docs

.github/workflows/32-bit-linux.yml

@@ -43,32 +43,18 @@ jobs:
       with:
         fetch-depth: 0
 
-    - name: Install Dependencies
-      uses: conda-incubator/setup-[email protected]
+    - name: Set up Conda
+      uses: ./.github/actions/setup-conda
       with:
-        mamba-version: "*"
-        channels: conda-forge
-        activate-environment: pandas-dev
-        channel-priority: ${{ matrix.os == 'macos-latest' && 'flexible' || 'strict' }}
         environment-file: ci/deps/${{ matrix.env_file }}
-        use-only-tar-bz2: true
-
-    # ImportError: 2): Library not loaded: @rpath/libssl.1.1.dylib
-    # Referenced from: /Users/runner/miniconda3/envs/pandas-dev/lib/libthrift.0.13.0.dylib
-    # Reason: image not found
-    - name: Upgrade pyarrow on MacOS
-      run: conda install -n pandas-dev -c conda-forge --no-update-deps pyarrow=6
-      if: ${{ matrix.os == 'macos-latest' }}
+        pyarrow-version: ${{ matrix.os == 'macos-latest' && '6' || '' }}
 
     - name: Build Pandas
       uses: ./.github/actions/build_pandas
 
     - name: Test
       run: ci/run_tests.sh
 
-    - name: Build Version
-      run: conda list
-
     - name: Publish test results
       uses: actions/upload-artifact@v3
       with:

.github/workflows/docbuild-and-upload.yml

@@ -11,7 +11,7 @@ repos:
     -   id: absolufy-imports
         files: ^pandas/
 -   repo: https://github.com/jendrikseipp/vulture
-    rev: 'v2.3'
+    rev: 'v2.4'
     hooks:
       - id: vulture
         entry: python scripts/run_vulture.py
@@ -60,7 +60,7 @@ repos:
     hooks:
     -   id: isort
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v2.32.0
+    rev: v2.32.1
     hooks:
     -   id: pyupgrade
         args: [--py38-plus]
@@ -75,7 +75,7 @@ repos:
         types: [text]  # overwrite types: [rst]
         types_or: [python, rst]
 -   repo: https://github.com/sphinx-contrib/sphinx-lint
-    rev: v0.4.1
+    rev: v0.6
     hooks:
     - id: sphinx-lint
 -   repo: https://github.com/asottile/yesqa

.github/workflows/macos-windows.yml

@@ -42,7 +42,7 @@
     // followed by the pip installed packages).
     "matrix": {
         "numpy": [],
-        "Cython": ["0.29.24"],
+        "Cython": ["0.29.30"],
         "matplotlib": [],
         "sqlalchemy": [],
         "scipy": [],

.pre-commit-config.yaml

@@ -7,6 +7,7 @@
 from pandas import (
     Categorical,
     DataFrame,
+    Index,
     MultiIndex,
     Series,
     Timestamp,
@@ -111,6 +112,18 @@ def time_copy_overhead_single_col(self, factor):
         self.df.groupby("key").apply(self.df_copy_function)
 
 
+class ApplyNonUniqueUnsortedIndex:
+    def setup(self):
+        # GH 46527
+        # unsorted and non-unique index
+        idx = np.arange(100)[::-1]
+        idx = Index(np.repeat(idx, 200), name="key")
+        self.df = DataFrame(np.random.randn(len(idx), 10), index=idx)
+
+    def time_groupby_apply_non_unique_unsorted_index(self):
+        self.df.groupby("key", group_keys=False).apply(lambda x: x)
+
+
 class Groups:
 
     param_names = ["key"]

asv_bench/asv.conf.json

@@ -86,4 +86,15 @@ def time_read_excel(self, engine):
         read_excel(fname, engine=engine)
 
 
+class ReadExcelNRows(ReadExcel):
+    def time_read_excel(self, engine):
+        if engine == "xlrd":
+            fname = self.fname_excel_xls
+        elif engine == "odf":
+            fname = self.fname_odf
+        else:
+            fname = self.fname_excel
+        read_excel(fname, engine=engine, nrows=10)
+
+
 from ..pandas_vb_common import setup  # noqa: F401 isort:skip

asv_bench/benchmarks/groupby.py

@@ -30,6 +30,10 @@ def setup(self):
         self.s_subset = self.s[::2]
         self.s_subset_no_cache = self.s[::2].copy()
 
+        mi = MultiIndex.from_product([rng, range(100)])
+        self.s2 = Series(np.random.randn(len(mi)), index=mi)
+        self.s2_subset = self.s2[::2].copy()
+
     def time_reindex_dates(self):
         self.df.reindex(self.rng_subset)
 
@@ -44,6 +48,10 @@ def time_reindex_multiindex_no_cache(self):
         # Copy to avoid MultiIndex._values getting cached
         self.s.reindex(self.s_subset_no_cache.index.copy())
 
+    def time_reindex_multiindex_no_cache_dates(self):
+        # Copy to avoid MultiIndex._values getting cached
+        self.s2_subset.reindex(self.s2.index.copy())
+
 
 class ReindexMethod:
 

asv_bench/benchmarks/io/excel.py

@@ -17,7 +17,7 @@ dependencies:
 
   # required dependencies
   - python-dateutil=2.8.1
-  - numpy=1.18.5
+  - numpy=1.19.5
   - pytz=2020.1
 
   # optional dependencies

asv_bench/benchmarks/reindex.py

@@ -12,7 +12,11 @@ you don't have to be an expert on pandas to do so! In fact,
 there are sections of the docs that are worse off after being written by
 experts. If something in the docs doesn't make sense to you, updating the
 relevant section after you figure it out is a great way to ensure it will help
-the next person.
+the next person. Please visit the `issues page <https://github.com/pandas-dev/pandas/issues?page=1&q=is%3Aopen+sort%3Aupdated-desc+label%3ADocs>`__
+for a full list of issues that are currently open regarding the
+Pandas documentation.
+
+
 
 .. contents:: Documentation:
    :local:

ci/deps/actions-38-minimum_versions.yaml

@@ -235,7 +235,7 @@ Dependencies
 ================================================================ ==========================
 Package                                                          Minimum supported version
 ================================================================ ==========================
-`NumPy <https://numpy.org>`__                                    1.18.5
+`NumPy <https://numpy.org>`__                                    1.19.5
 `python-dateutil <https://dateutil.readthedocs.io/en/stable/>`__ 2.8.1
 `pytz <https://pypi.org/project/pytz/>`__                        2020.1
 ================================================================ ==========================

doc/source/development/contributing_documentation.rst

@@ -143,8 +143,8 @@ returned.
 
 Calculating a given statistic (e.g. ``mean`` age) *for each category in
 a column* (e.g. male/female in the ``Sex`` column) is a common pattern.
-The ``groupby`` method is used to support this type of operations. More
-general, this fits in the more general ``split-apply-combine`` pattern:
+The ``groupby`` method is used to support this type of operations. This
+fits in the more general ``split-apply-combine`` pattern:
 
 -  **Split** the data into groups
 -  **Apply** a function to each group independently

doc/source/getting_started/install.rst

@@ -42,6 +42,7 @@ Exceptions and warnings
    errors.ParserWarning
    errors.PerformanceWarning
    errors.SettingWithCopyError
+   errors.SettingWithCopyWarning
    errors.SpecificationError
    errors.UnsortedIndexError
    errors.UnsupportedFunctionCall

doc/source/getting_started/intro_tutorials/06_calculate_statistics.rst

@@ -15,13 +15,16 @@ including other versions of pandas.
 Fixed regressions
 ~~~~~~~~~~~~~~~~~
 - Fixed regression in :meth:`DataFrame.replace` when the replacement value was explicitly ``None`` when passed in a dictionary to ``to_replace`` also casting other columns to object dtype even when there were no values to replace (:issue:`46634`)
+- Fixed regression when setting values with :meth:`DataFrame.loc` updating :class:`RangeIndex` when index was set as new column and column was updated afterwards (:issue:`47128`)
 - Fixed regression in :meth:`DataFrame.nsmallest` led to wrong results when ``np.nan`` in the sorting column (:issue:`46589`)
 - Fixed regression in :func:`read_fwf` raising ``ValueError`` when ``widths`` was specified with ``usecols`` (:issue:`46580`)
+- Fixed regression in :func:`concat` not sorting columns for mixed column names (:issue:`47127`)
 - Fixed regression in :meth:`.Groupby.transform` and :meth:`.Groupby.agg` failing with ``engine="numba"`` when the index was a :class:`MultiIndex` (:issue:`46867`)
 - Fixed regression is :meth:`.Styler.to_latex` and :meth:`.Styler.to_html` where ``buf`` failed in combination with ``encoding`` (:issue:`47053`)
 - Fixed regression in :func:`read_csv` with ``index_col=False`` identifying first row as index names when ``header=None`` (:issue:`46955`)
 - Fixed regression in :meth:`.DataFrameGroupBy.agg` when used with list-likes or dict-likes and ``axis=1`` that would give incorrect results; now raises ``NotImplementedError`` (:issue:`46995`)
 - Fixed regression in :meth:`DataFrame.resample` and :meth:`DataFrame.rolling` when used with list-likes or dict-likes and ``axis=1`` that would raise an unintuitive error message; now raises ``NotImplementedError`` (:issue:`46904`)
+- Fixed regression in :func:`read_excel` returning ints as floats on certain input sheets (:issue:`46988`)
 - Fixed regression in :meth:`DataFrame.shift` when ``axis`` is ``columns`` and ``fill_value`` is absent, ``freq`` is ignored (:issue:`47039`)
 
 .. ---------------------------------------------------------------------------
@@ -30,6 +33,7 @@ Fixed regressions
 
 Bug fixes
 ~~~~~~~~~
+- Bug in :meth:`pd.eval`, :meth:`DataFrame.eval` and :meth:`DataFrame.query` where passing empty ``local_dict`` or ``global_dict`` was treated as passing ``None`` (:issue:`47084`)
 - Most I/O methods do no longer suppress ``OSError`` and ``ValueError`` when closing file handles (:issue:`47136`)
 -