Merge remote-tracking branch 'upstream/main' into multi-bug

Author: KingOfTheShow69

.circleci/setup_env.sh

@@ -1,47 +1,16 @@
 #!/bin/bash -e
 
-# edit the locale file if needed
-if [[ "$(uname)" == "Linux" && -n "$LC_ALL" ]]; then
-    echo "Adding locale to the first line of pandas/__init__.py"
-    rm -f pandas/__init__.pyc
-    SEDC="3iimport locale\nlocale.setlocale(locale.LC_ALL, '$LC_ALL')\n"
-    sed -i "$SEDC" pandas/__init__.py
-
-    echo "[head -4 pandas/__init__.py]"
-    head -4 pandas/__init__.py
-    echo
-fi
+echo "Install Mambaforge"
+MAMBA_URL="https://github.com/conda-forge/miniforge/releases/download/4.14.0-0/Mambaforge-4.14.0-0-Linux-aarch64.sh"
+echo "Downloading $MAMBA_URL"
+wget -q $MAMBA_URL -O minimamba.sh
+chmod +x minimamba.sh
 
+MAMBA_DIR="$HOME/miniconda3"
+rm -rf $MAMBA_DIR
+./minimamba.sh -b -p $MAMBA_DIR
 
-MINICONDA_DIR=/usr/local/miniconda
-if [ -e $MINICONDA_DIR ] && [ "$BITS32" != yes ]; then
-    echo "Found Miniconda installation at $MINICONDA_DIR"
-else
-    echo "Install Miniconda"
-    DEFAULT_CONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest"
-    if [[ "$(uname -m)" == 'aarch64' ]]; then
-        CONDA_URL="https://github.com/conda-forge/miniforge/releases/download/4.10.1-4/Miniforge3-4.10.1-4-Linux-aarch64.sh"
-    elif [[ "$(uname)" == 'Linux' ]]; then
-        if [[ "$BITS32" == "yes" ]]; then
-            CONDA_URL="$DEFAULT_CONDA_URL-Linux-x86.sh"
-        else
-            CONDA_URL="$DEFAULT_CONDA_URL-Linux-x86_64.sh"
-        fi
-    elif [[ "$(uname)" == 'Darwin' ]]; then
-        CONDA_URL="$DEFAULT_CONDA_URL-MacOSX-x86_64.sh"
-    else
-      echo "OS $(uname) not supported"
-      exit 1
-    fi
-    echo "Downloading $CONDA_URL"
-    wget -q $CONDA_URL -O miniconda.sh
-    chmod +x miniconda.sh
-
-    MINICONDA_DIR="$HOME/miniconda3"
-    rm -rf $MINICONDA_DIR
-    ./miniconda.sh -b -p $MINICONDA_DIR
-fi
-export PATH=$MINICONDA_DIR/bin:$PATH
+export PATH=$MAMBA_DIR/bin:$PATH
 
 echo
 echo "which conda"
@@ -51,7 +20,7 @@ echo
 echo "update conda"
 conda config --set ssl_verify false
 conda config --set quiet true --set always_yes true --set changeps1 false
-conda install -y -c conda-forge -n base 'mamba>=0.21.2' pip setuptools
+mamba install -y -c conda-forge -n base pip setuptools
 
 echo "conda info -a"
 conda info -a
@@ -70,11 +39,6 @@ time mamba env update -n pandas-dev --file="${ENV_FILE}"
 echo "conda list -n pandas-dev"
 conda list -n pandas-dev
 
-if [[ "$BITS32" == "yes" ]]; then
-    # activate 32-bit compiler
-    export CONDA_BUILD=1
-fi
-
 echo "activate pandas-dev"
 source activate pandas-dev
 
@@ -90,15 +54,9 @@ if pip list | grep -q ^pandas; then
     pip uninstall -y pandas || true
 fi
 
-if [ "$(conda list -f qt --json)" != [] ]; then
-    echo
-    echo "remove qt"
-    echo "causes problems with the clipboard, we use xsel for that"
-    conda remove qt -y --force || true
-fi
-
 echo "Build extensions"
-python setup.py build_ext -q -j3
+# GH 47305: Parallel build can causes flaky ImportError from pandas/_libs/tslibs
+python setup.py build_ext -q -j1
 
 echo "Install pandas"
 python -m pip install --no-build-isolation --no-use-pep517 -e .

.github/workflows/docbuild-and-upload.yml

@@ -67,7 +67,7 @@ jobs:
         echo "${{ secrets.server_ssh_key }}" > ~/.ssh/id_rsa
         chmod 600 ~/.ssh/id_rsa
         echo "${{ secrets.server_ip }} ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBE1Kkopomm7FHG5enATf7SgnpICZ4W2bw+Ho+afqin+w7sMcrsa0je7sbztFAV8YchDkiBKnWTG4cRT+KZgZCaY=" > ~/.ssh/known_hosts
-      if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+      if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/'))
 
     - name: Copy cheatsheets into site directory
       run: cp doc/cheatsheet/Pandas_Cheat_Sheet* web/build/

.pre-commit-config.yaml

@@ -22,7 +22,7 @@ repos:
     hooks:
     -   id: black
 -   repo: https://github.com/codespell-project/codespell
-    rev: v2.1.0
+    rev: v2.2.1
     hooks:
     -   id: codespell
         types_or: [python, rst, markdown]

Dockerfile

@@ -1,4 +1,4 @@
-FROM quay.io/condaforge/mambaforge
+FROM quay.io/condaforge/mambaforge:4.13.0-1
 
 # if you forked pandas, you can pass in your own GitHub username to use your fork
 # i.e. gh_username=myname

asv_bench/benchmarks/groupby.py

@@ -5,6 +5,7 @@
 import numpy as np
 
 from pandas import (
+    NA,
     Categorical,
     DataFrame,
     Index,
@@ -560,6 +561,46 @@ def time_frame_agg(self, dtype, method):
         self.df.groupby("key").agg(method)
 
 
+class GroupByCythonAggEaDtypes:
+    """
+    Benchmarks specifically targeting our cython aggregation algorithms
+    (using a big enough dataframe with simple key, so a large part of the
+    time is actually spent in the grouped aggregation).
+    """
+
+    param_names = ["dtype", "method"]
+    params = [
+        ["Float64", "Int64", "Int32"],
+        [
+            "sum",
+            "prod",
+            "min",
+            "max",
+            "mean",
+            "median",
+            "var",
+            "first",
+            "last",
+            "any",
+            "all",
+        ],
+    ]
+
+    def setup(self, dtype, method):
+        N = 1_000_000
+        df = DataFrame(
+            np.random.randint(0, high=100, size=(N, 10)),
+            columns=list("abcdefghij"),
+            dtype=dtype,
+        )
+        df.loc[list(range(1, N, 5)), list("abcdefghij")] = NA
+        df["key"] = np.random.randint(0, 100, size=N)
+        self.df = df
+
+    def time_frame_agg(self, dtype, method):
+        self.df.groupby("key").agg(method)
+
+
 class Cumulative:
     param_names = ["dtype", "method"]
     params = [

asv_bench/benchmarks/multiindex_object.py

@@ -3,9 +3,11 @@
 import numpy as np
 
 from pandas import (
+    NA,
     DataFrame,
     MultiIndex,
     RangeIndex,
+    Series,
     date_range,
 )
 
@@ -255,4 +257,31 @@ def time_operation(self, index_structure, dtype, method):
         getattr(self.left, method)(self.right)
 
 
+class Unique:
+    params = [
+        (("Int64", NA), ("int64", 0)),
+    ]
+    param_names = ["dtype_val"]
+
+    def setup(self, dtype_val):
+        level = Series(
+            [1, 2, dtype_val[1], dtype_val[1]] + list(range(1_000_000)),
+            dtype=dtype_val[0],
+        )
+        self.midx = MultiIndex.from_arrays([level, level])
+
+        level_dups = Series(
+            [1, 2, dtype_val[1], dtype_val[1]] + list(range(500_000)) * 2,
+            dtype=dtype_val[0],
+        )
+
+        self.midx_dups = MultiIndex.from_arrays([level_dups, level_dups])
+
+    def time_unique(self, dtype_val):
+        self.midx.unique()
+
+    def time_unique_dups(self, dtype_val):
+        self.midx_dups.unique()
+
+
 from .pandas_vb_common import setup  # noqa: F401 isort:skip

ci/deps/actions-310.yaml

@@ -43,7 +43,7 @@ dependencies:
   - pyreadstat
   - python-snappy
   - pyxlsb
-  - s3fs
+  - s3fs>=2021.08.0
   - scipy
   - sqlalchemy
   - tabulate

ci/deps/actions-38-downstream_compat.yaml

@@ -44,7 +44,7 @@ dependencies:
   - pytables
   - python-snappy
   - pyxlsb
-  - s3fs
+  - s3fs>=2021.08.0
   - scipy
   - sqlalchemy
   - tabulate

ci/deps/actions-38-minimum_versions.yaml

@@ -26,10 +26,10 @@ dependencies:
   - bottleneck=1.3.2
   - brotlipy=0.7.0
   - fastparquet=0.4.0
-  - fsspec=2021.05.0
+  - fsspec=2021.07.0
   - html5lib=1.1
   - hypothesis=6.13.0
-  - gcsfs=2021.05.0
+  - gcsfs=2021.07.0
   - jinja2=3.0.0
   - lxml=4.6.3
   - matplotlib=3.3.2
@@ -45,7 +45,7 @@ dependencies:
   - pytables=3.6.1
   - python-snappy=0.6.0
   - pyxlsb=1.0.8
-  - s3fs=2021.05.0
+  - s3fs=2021.08.0
   - scipy=1.7.1
   - sqlalchemy=1.4.16
   - tabulate=0.8.9

ci/deps/actions-38.yaml

@@ -43,7 +43,7 @@ dependencies:
   - pytables
   - python-snappy
   - pyxlsb
-  - s3fs
+  - s3fs>=2021.08.0
   - scipy
   - sqlalchemy
   - tabulate

ci/deps/actions-39.yaml

@@ -43,7 +43,7 @@ dependencies:
   - pytables
   - python-snappy
   - pyxlsb
-  - s3fs
+  - s3fs>=2021.08.0
   - scipy
   - sqlalchemy
   - tabulate

ci/deps/circle-38-arm64.yaml

@@ -44,7 +44,7 @@ dependencies:
   - pytables
   - python-snappy
   - pyxlsb
-  - s3fs
+  - s3fs>=2021.08.0
   - scipy
   - sqlalchemy
   - tabulate

doc/source/development/contributing.rst

@@ -194,30 +194,10 @@ Doing 'git status' again should give something like::
     #       modified:   /relative/path/to/file-you-added.py
     #
 
-Finally, commit your changes to your local repository with an explanatory message. pandas
-uses a convention for commit message prefixes and layout.  Here are
-some common prefixes along with general guidelines for when to use them:
+Finally, commit your changes to your local repository with an explanatory commit
+message::
 
-* ENH: Enhancement, new functionality
-* BUG: Bug fix
-* DOC: Additions/updates to documentation
-* TST: Additions/updates to tests
-* BLD: Updates to the build process/scripts
-* PERF: Performance improvement
-* TYP: Type annotations
-* CLN: Code cleanup
-
-The following defines how a commit message should be structured.  Please reference the
-relevant GitHub issues in your commit message using GH1234 or #1234.  Either style
-is fine, but the former is generally preferred:
-
-* a subject line with ``< 80`` chars.
-* One blank line.
-* Optionally, a commit message body.
-
-Now you can commit your changes in your local repository::
-
-    git commit -m
+    git commit -m "your commit message goes here"
 
 .. _contributing.push-code:
 
@@ -262,16 +242,28 @@ double check your branch changes against the branch it was based on:
 Finally, make the pull request
 ------------------------------
 
-If everything looks good, you are ready to make a pull request.  A pull request is how
+If everything looks good, you are ready to make a pull request. A pull request is how
 code from a local repository becomes available to the GitHub community and can be looked
-at and eventually merged into the main version.  This pull request and its associated
+at and eventually merged into the main version. This pull request and its associated
 changes will eventually be committed to the main branch and available in the next
-release.  To submit a pull request:
+release. To submit a pull request:
 
 #. Navigate to your repository on GitHub
-#. Click on the ``Pull Request`` button
+#. Click on the ``Compare & pull request`` button
 #. You can then click on ``Commits`` and ``Files Changed`` to make sure everything looks
    okay one last time
+#. Write a descriptive title that includes prefixes. pandas uses a convention for title
+   prefixes. Here are some common ones along with general guidelines for when to use them:
+
+    * ENH: Enhancement, new functionality
+    * BUG: Bug fix
+    * DOC: Additions/updates to documentation
+    * TST: Additions/updates to tests
+    * BLD: Updates to the build process/scripts
+    * PERF: Performance improvement
+    * TYP: Type annotations
+    * CLN: Code cleanup
+
 #. Write a description of your changes in the ``Preview Discussion`` tab
 #. Click ``Send Pull Request``.
 

doc/source/development/contributing_codebase.rst

@@ -75,6 +75,11 @@ If you want to run checks on all recently committed files on upstream/main you c
 
 without needing to have done ``pre-commit install`` beforehand.
 
+.. note::
+
+    You may want to periodically run ``pre-commit gc``, to clean up repos
+    which are no longer used.
+
 .. note::
 
     If you have conflicting installations of ``virtualenv``, then you may get an