Merge remote-tracking branch 'upstream/main' into bisect

Author: simonjayhawkins

azure-pipelines.yml

@@ -45,7 +45,7 @@ jobs:
         /opt/python/cp38-cp38/bin/python -m venv ~/virtualenvs/pandas-dev && \
         . ~/virtualenvs/pandas-dev/bin/activate && \
         python -m pip install --no-deps -U pip wheel 'setuptools<60.0.0' && \
-        pip install cython numpy python-dateutil pytz pytest pytest-xdist hypothesis pytest-azurepipelines && \
+        pip install cython numpy python-dateutil pytz pytest pytest-xdist pytest-asyncio hypothesis && \
         python setup.py build_ext -q -j2 && \
         python -m pip install --no-build-isolation -e . && \
         pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas --junitxml=test-data.xml"

ci/deps/actions-310-numpydev.yaml

@@ -9,6 +9,7 @@ dependencies:
   - pytest-cov
   - pytest-xdist>=1.31
   - hypothesis>=5.5.3
+  - pytest-asyncio
 
   # pandas dependencies
   - python-dateutil

ci/deps/actions-310.yaml

@@ -42,10 +42,10 @@ dependencies:
   - pytables
   - pyarrow
   - pyreadstat
+  - python-snappy
   - pyxlsb
   - s3fs
   - scipy
-  - snappy
   - sqlalchemy
   - tabulate
   - xarray

ci/deps/actions-38-downstream_compat.yaml

@@ -38,14 +38,14 @@ dependencies:
   - odfpy
   - pandas-gbq
   - psycopg2
-  - pymysql
-  - pytables
   - pyarrow
+  - pymysql
   - pyreadstat
+  - pytables
+  - python-snappy
   - pyxlsb
   - s3fs
   - scipy
-  - snappy
   - sqlalchemy
   - tabulate
   - xarray

ci/deps/actions-38-minimum_versions.yaml

@@ -40,14 +40,14 @@ dependencies:
   - openpyxl=3.0.3
   - pandas-gbq=0.14.0
   - psycopg2=2.8.4
-  - pymysql=0.10.1
-  - pytables=3.6.1
   - pyarrow=1.0.1
+  - pymysql=0.10.1
   - pyreadstat=1.1.0
+  - pytables=3.6.1
+  - python-snappy=0.6.0
   - pyxlsb=1.0.6
   - s3fs=0.4.0
   - scipy=1.4.1
-  - snappy=1.1.8
   - sqlalchemy=1.4.0
   - tabulate=0.8.7
   - xarray=0.15.1

ci/deps/actions-38.yaml

@@ -37,14 +37,14 @@ dependencies:
   - odfpy
   - pandas-gbq
   - psycopg2
-  - pymysql
-  - pytables
   - pyarrow
+  - pymysql
   - pyreadstat
+  - pytables
+  - python-snappy
   - pyxlsb
   - s3fs
   - scipy
-  - snappy
   - sqlalchemy
   - tabulate
   - xarray

ci/deps/actions-39.yaml

@@ -38,13 +38,13 @@ dependencies:
   - pandas-gbq
   - psycopg2
   - pymysql
-  - pytables
   - pyarrow
   - pyreadstat
+  - pytables
+  - python-snappy
   - pyxlsb
   - s3fs
   - scipy
-  - snappy
   - sqlalchemy
   - tabulate
   - xarray

ci/deps/circle-38-arm64.yaml

@@ -9,6 +9,7 @@ dependencies:
   - pytest>=6.0
   - pytest-xdist>=1.31
   - hypothesis>=5.5.3
+  - pytest-asyncio
 
   # pandas dependencies
   - botocore>=1.11

doc/source/getting_started/install.rst

@@ -411,6 +411,6 @@ Compression
 Dependency                Minimum Version    Notes
 ========================= ================== =============================================================
 brotli                    0.7.0              Brotli compression
-snappy                    1.1.8              Snappy compression
+python-snappy             0.6.0              Snappy compression
 Zstandard                 0.15.2             Zstandard compression
 ========================= ================== =============================================================

doc/source/user_guide/enhancingperf.rst

@@ -350,6 +350,28 @@ a larger amount of data points (e.g. 1+ million).
    In [6]: %timeit roll.apply(f, engine='cython', raw=True)
    3.92 s ± 59 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)
 
+If your compute hardware contains multiple CPUs, the largest performance gain can be realized by setting ``parallel`` to ``True``
+to leverage more than 1 CPU. Internally, pandas leverages numba to parallelize computations over the columns of a :class:`DataFrame`;
+therefore, this performance benefit is only beneficial for a :class:`DataFrame` with a large number of columns.
+
+.. code-block:: ipython
+
+   In [1]: import numba
+
+   In [2]: numba.set_num_threads(1)
+
+   In [3]: df = pd.DataFrame(np.random.randn(10_000, 100))
+
+   In [4]: roll = df.rolling(100)
+
+   In [5]: %timeit roll.mean(engine="numba", engine_kwargs={"parallel": True})
+   347 ms ± 26 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)
+
+   In [6]: numba.set_num_threads(2)
+
+   In [7]: %timeit roll.mean(engine="numba", engine_kwargs={"parallel": True})
+   201 ms ± 2.97 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)
+
 Custom Function Examples
 ~~~~~~~~~~~~~~~~~~~~~~~~
 

doc/source/whatsnew/v1.4.2.rst

@@ -18,7 +18,8 @@ Fixed regressions
 - Fixed regression in :func:`read_csv` killing python process when invalid file input was given for ``engine="c"`` (:issue:`45957`)
 - Fixed memory performance regression in :meth:`Series.fillna` when called on a :class:`DataFrame` column with ``inplace=True`` (:issue:`46149`)
 - Provided an alternative solution for passing custom Excel formats in :meth:`.Styler.to_excel`, which was a regression based on stricter CSS validation. Examples available in the documentation for :meth:`.Styler.format` (:issue:`46152`)
-- Fixed regression in :meth:`DataFrame.loc.__setitem__` losing :class:`MultiIndex` names if :class:`DataFrame`  was empty before (:issue:`46317`)
+- Fixed regression in :meth:`DataFrame.replace` when a replacement value was also a target for replacement (:issue:`46306`)
+- Fixed regression when setting values with :meth:`DataFrame.loc` losing :class:`MultiIndex` names if :class:`DataFrame`  was empty before (:issue:`46317`)
 -
 
 .. ---------------------------------------------------------------------------

doc/source/whatsnew/v1.5.0.rst

@@ -311,6 +311,7 @@ Performance improvements
 - Performance improvement in :meth:`.GroupBy.transform` when broadcasting values for user-defined functions (:issue:`45708`)
 - Performance improvement in :meth:`.GroupBy.transform` for user-defined functions when only a single group exists (:issue:`44977`)
 - Performance improvement in :meth:`MultiIndex.get_locs` (:issue:`45681`, :issue:`46040`)
+- Performance improvement in :attr:`MultiIndex.values` when the MultiIndex contains levels of type DatetimeIndex, TimedeltaIndex or ExtensionDtypes (:issue:`46288`)
 - Performance improvement in :func:`merge` when left and/or right are empty (:issue:`45838`)
 - Performance improvement in :meth:`DataFrame.join` when left and/or right are empty (:issue:`46015`)
 - Performance improvement in :meth:`DataFrame.reindex` and :meth:`Series.reindex` when target is a :class:`MultiIndex` (:issue:`46235`)
@@ -394,6 +395,7 @@ Indexing
 - Bug in :meth:`Series.mask` with ``inplace=True`` or setting values with a boolean mask with small integer dtypes incorrectly raising (:issue:`45750`)
 - Bug in :meth:`DataFrame.mask` with ``inplace=True`` and ``ExtensionDtype`` columns incorrectly raising (:issue:`45577`)
 - Bug in getting a column from a DataFrame with an object-dtype row index with datetime-like values: the resulting Series now preserves the exact object-dtype Index from the parent DataFrame (:issue:`42950`)
+- Bug in :meth:`DataFrame.__getattribute__` raising ``AttributeError`` if columns have ``"string"`` dtype (:issue:`46185`)
 - Bug in indexing on a :class:`DatetimeIndex` with a ``np.str_`` key incorrectly raising (:issue:`45580`)
 - Bug in :meth:`CategoricalIndex.get_indexer` when index contains ``NaN`` values, resulting in elements that are in target but not present in the index to be mapped to the index of the NaN element, instead of -1 (:issue:`45361`)
 - Bug in setting large integer values into :class:`Series` with ``float32`` or ``float16`` dtype incorrectly altering these values instead of coercing to ``float64`` dtype (:issue:`45844`)
@@ -453,6 +455,7 @@ Groupby/resample/rolling
 - Bug in :meth:`DataFrameGroupby.cumsum` with ``skipna=False`` giving incorrect results (:issue:`46216`)
 - Bug in :meth:`.GroupBy.cumsum` with ``timedelta64[ns]`` dtype failing to recognize ``NaT`` as a null value (:issue:`46216`)
 - Bug in :meth:`GroupBy.cummin` and :meth:`GroupBy.cummax` with nullable dtypes incorrectly altering the original data in place (:issue:`46220`)
+- Bug in :meth:`GroupBy.cummax` with ``int64`` dtype with leading value being the smallest possible int64 (:issue:`46382`)
 -
 
 Reshaping

pandas/_libs/groupby.pyx

@@ -989,13 +989,18 @@ cdef inline bint _treat_as_na(numeric_object_t val, bint is_datetimelike) nogil:
         return False
 
 
-cdef numeric_t _get_min_or_max(numeric_t val, bint compute_max):
+cdef numeric_t _get_min_or_max(numeric_t val, bint compute_max, bint is_datetimelike):
     """
     Find either the min or the max supported by numeric_t; 'val' is a placeholder
     to effectively make numeric_t an argument.
     """
     if numeric_t is int64_t:
-        return -_int64_max if compute_max else util.INT64_MAX
+        if compute_max and is_datetimelike:
+            return -_int64_max
+        # Note(jbrockmendel) 2022-03-15 for reasons unknown, using util.INT64_MIN
+        #  instead of NPY_NAT here causes build warnings and failure in
+        #  test_cummax_i8_at_implementation_bound
+        return NPY_NAT if compute_max else util.INT64_MAX
     elif numeric_t is int32_t:
         return util.INT32_MIN if compute_max else util.INT32_MAX
     elif numeric_t is int16_t:
@@ -1395,7 +1400,7 @@ cdef group_min_max(
     nobs = np.zeros((<object>out).shape, dtype=np.int64)
 
     group_min_or_max = np.empty_like(out)
-    group_min_or_max[:] = _get_min_or_max(<iu_64_floating_t>0, compute_max)
+    group_min_or_max[:] = _get_min_or_max(<iu_64_floating_t>0, compute_max, is_datetimelike)
 
     if iu_64_floating_t is int64_t:
         # TODO: only if is_datetimelike?
@@ -1564,7 +1569,7 @@ cdef group_cummin_max(
         bint isna_entry
 
     accum = np.empty((ngroups, (<object>values).shape[1]), dtype=values.dtype)
-    accum[:] = _get_min_or_max(<iu_64_floating_t>0, compute_max)
+    accum[:] = _get_min_or_max(<iu_64_floating_t>0, compute_max, is_datetimelike)
 
     na_val = _get_na_val(<iu_64_floating_t>0, is_datetimelike)