Merge branch 'master' into stringarray-nan

Author: lithomas1

.github/workflows/ci.yml

@@ -136,8 +136,11 @@ jobs:
         echo "${{ secrets.server_ip }} ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBE1Kkopomm7FHG5enATf7SgnpICZ4W2bw+Ho+afqin+w7sMcrsa0je7sbztFAV8YchDkiBKnWTG4cRT+KZgZCaY=" > ~/.ssh/known_hosts
       if: ${{github.event_name == 'push' && github.ref == 'refs/heads/master'}}
 
+    - name: Copy cheatsheets into site directory
+      run: cp doc/cheatsheet/Pandas_Cheat_Sheet* web/build/
+
     - name: Upload web
-      run: rsync -az --delete --exclude='pandas-docs' --exclude='docs' --exclude='Pandas_Cheat_Sheet*' web/build/ docs@${{ secrets.server_ip }}:/usr/share/nginx/pandas
+      run: rsync -az --delete --exclude='pandas-docs' --exclude='docs' web/build/ docs@${{ secrets.server_ip }}:/usr/share/nginx/pandas
       if: ${{github.event_name == 'push' && github.ref == 'refs/heads/master'}}
 
     - name: Upload dev docs
@@ -146,6 +149,7 @@ jobs:
 
     - name: Move docs into site directory
       run: mv doc/build/html web/build/docs
+
     - name: Save website as an artifact
       uses: actions/upload-artifact@v2
       with:

.github/workflows/pre-commit.yml

@@ -16,4 +16,6 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - uses: actions/setup-python@v2
+      with:
+        python-version: '3.9.7'
     - uses: pre-commit/[email protected]

.github/workflows/python-dev.yml

@@ -17,7 +17,7 @@ env:
   PANDAS_CI: 1
   PATTERN: "not slow and not network and not clipboard"
   COVERAGE: true
-  PYTEST_TARGET:  pandas
+  PYTEST_TARGET: pandas
 
 jobs:
   build:
@@ -28,13 +28,10 @@ jobs:
         os: [ubuntu-latest, macOS-latest, windows-latest]
 
     name: actions-310-dev
-    timeout-minutes: 60
-
-    env:
-      NUMPY_WHEELS_AVAILABLE: ${{ matrix.os == 'ubuntu-latest' }}
+    timeout-minutes: 80
 
     concurrency:
-      group: ${{ github.ref }}-${{ matrix.os }}-dev
+      group: ${{ github.ref }}-${{ matrix.os }}-${{ matrix.pytest_target }}-dev
       cancel-in-progress: ${{github.event_name == 'pull_request'}}
 
     steps:
@@ -51,11 +48,7 @@ jobs:
       shell: bash
       run: |
         python -m pip install --upgrade pip setuptools wheel
-        if [[ "$NUMPY_WHEELS_AVAILABLE" == "true" ]]; then
-          pip install -i https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy
-        else
-          pip install git+https://github.com/numpy/numpy.git
-        fi
+        pip install -i https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy
         pip install git+https://github.com/nedbat/coveragepy.git
         pip install cython python-dateutil pytz hypothesis pytest>=6.2.5 pytest-xdist pytest-cov
         pip list

.github/workflows/sdist.yml

@@ -53,6 +53,7 @@ jobs:
     - uses: conda-incubator/setup-miniconda@v2
       with:
         activate-environment: pandas-sdist
+        channels: conda-forge
         python-version: '${{ matrix.python-version }}'
 
     - name: Install pandas from sdist

.pre-commit-config.yaml

@@ -9,7 +9,7 @@ repos:
     -   id: absolufy-imports
         files: ^pandas/
 -   repo: https://github.com/python/black
-    rev: 21.9b0
+    rev: 21.10b0
     hooks:
     -   id: black
 -   repo: https://github.com/codespell-project/codespell
@@ -39,10 +39,11 @@ repos:
     rev: 3.9.2
     hooks:
     -   id: flake8
-        additional_dependencies:
-            - flake8-comprehensions==3.1.0
-            - flake8-bugbear==21.3.2
-            - pandas-dev-flaker==0.2.0
+        additional_dependencies: &flake8_dependencies
+        - flake8==3.9.2
+        - flake8-comprehensions==3.1.0
+        - flake8-bugbear==21.3.2
+        - pandas-dev-flaker==0.2.0
     -   id: flake8
         alias: flake8-cython
         name: flake8 (cython)
@@ -73,14 +74,10 @@ repos:
         types: [text]  # overwrite types: [rst]
         types_or: [python, rst]
 -   repo: https://github.com/asottile/yesqa
-    rev: v1.2.3
+    rev: v1.3.0
     hooks:
     -   id: yesqa
-        additional_dependencies:
-            - flake8==3.9.2
-            - flake8-comprehensions==3.1.0
-            - flake8-bugbear==21.3.2
-            - pandas-dev-flaker==0.2.0
+        additional_dependencies: *flake8_dependencies
 -   repo: local
     hooks:
     -   id: pyright

MANIFEST.in

@@ -33,6 +33,7 @@ global-exclude *.xlsb
 global-exclude *.xlsm
 global-exclude *.xlsx
 global-exclude *.xpt
+global-exclude *.cpt
 global-exclude *.xz
 global-exclude *.zip
 global-exclude *~

asv_bench/asv.conf.json

@@ -39,7 +39,7 @@
     // followed by the pip installed packages).
     "matrix": {
         "numpy": [],
-        "Cython": ["0.29.21"],
+        "Cython": ["0.29.24"],
         "matplotlib": [],
         "sqlalchemy": [],
         "scipy": [],

asv_bench/benchmarks/groupby.py

@@ -443,7 +443,7 @@ class GroupByMethods:
             "var",
         ],
         ["direct", "transformation"],
-        [1, 2, 5, 10],
+        [1, 5],
     ]
 
     def setup(self, dtype, method, application, ncols):
@@ -455,6 +455,7 @@ def setup(self, dtype, method, application, ncols):
             raise NotImplementedError
 
         if application == "transformation" and method in [
+            "describe",
             "head",
             "tail",
             "unique",
@@ -464,7 +465,12 @@ def setup(self, dtype, method, application, ncols):
             # DataFrameGroupBy doesn't have these methods
             raise NotImplementedError
 
-        ngroups = 1000
+        if method == "describe":
+            ngroups = 20
+        elif method in ["mad", "skew"]:
+            ngroups = 100
+        else:
+            ngroups = 1000
         size = ngroups * 2
         rng = np.arange(ngroups).reshape(-1, 1)
         rng = np.broadcast_to(rng, (len(rng), ncols))
@@ -491,9 +497,6 @@ def setup(self, dtype, method, application, ncols):
             cols = cols[0]
 
         if application == "transformation":
-            if method == "describe":
-                raise NotImplementedError
-
             self.as_group_method = lambda: df.groupby("key")[cols].transform(method)
             self.as_field_method = lambda: df.groupby(cols)["key"].transform(method)
         else:

asv_bench/benchmarks/index_object.py

@@ -86,6 +86,12 @@ def time_iter_dec(self):
         for _ in self.idx_dec:
             pass
 
+    def time_sort_values_asc(self):
+        self.idx_inc.sort_values()
+
+    def time_sort_values_des(self):
+        self.idx_inc.sort_values(ascending=False)
+
 
 class IndexEquals:
     def setup(self):

asv_bench/benchmarks/inference.py

@@ -277,6 +277,16 @@ def time_dup_string_tzoffset_dates(self, cache):
         to_datetime(self.dup_string_with_tz, cache=cache)
 
 
+# GH 43901
+class ToDatetimeInferDatetimeFormat:
+    def setup(self):
+        rng = date_range(start="1/1/2000", periods=100000, freq="H")
+        self.strings = rng.strftime("%Y-%m-%d %H:%M:%S").tolist()
+
+    def time_infer_datetime_format(self):
+        to_datetime(self.strings, infer_datetime_format=True)
+
+
 class ToTimedelta:
     def setup(self):
         self.ints = np.random.randint(0, 60, size=10000)

asv_bench/benchmarks/io/csv.py

@@ -525,4 +525,14 @@ def time_to_datetime_format_DD_MM_YYYY(self, cache_dates):
         to_datetime(df["date"], cache=cache_dates, format="%d-%m-%Y")
 
 
+class ReadCSVIndexCol(StringIORewind):
+    def setup(self):
+        count_elem = 100_000
+        data = "a,b\n" + "1,2\n" * count_elem
+        self.StringIO_input = StringIO(data)
+
+    def time_read_csv_index_col(self):
+        read_csv(self.StringIO_input, index_col="a")
+
+
 from ..pandas_vb_common import setup  # noqa: F401 isort:skip

asv_bench/benchmarks/pandas_vb_common.py

@@ -17,7 +17,7 @@
 try:
     import pandas._testing as tm
 except ImportError:
-    import pandas.util.testing as tm  # noqa
+    import pandas.util.testing as tm  # noqa:F401
 
 
 numeric_dtypes = [