Merge remote-tracking branch 'upstream/master' into move-metadata-to-cfg

Author: fangchenli

.pre-commit-config.yaml

@@ -60,11 +60,11 @@ repos:
         entry: |
             (?x)
             # Check for imports from pandas.core.common instead of `import pandas.core.common as com`
-            from\ pandas\.core\.common\ import|
-            from\ pandas\.core\ import\ common|
+            from\ pandas\.core\.common\ import
+            |from\ pandas\.core\ import\ common
 
             # Check for imports from collections.abc instead of `from collections import abc`
-            from\ collections\.abc\ import
+            |from\ collections\.abc\ import
 
     -   id: non-standard-numpy.random-related-imports
         name: Check for non-standard numpy.random-related imports excluding pandas/_testing.py
@@ -73,24 +73,24 @@ repos:
         entry: |
             (?x)
             # Check for imports from np.random.<method> instead of `from numpy import random` or `from numpy.random import <method>`
-            from\ numpy\ import\ random|
-            from\ numpy.random\ import
+            from\ numpy\ import\ random
+            |from\ numpy.random\ import
         types: [python]
     -   id: non-standard-imports-in-tests
         name: Check for non-standard imports in test suite
         language: pygrep
         entry: |
             (?x)
             # Check for imports from pandas._testing instead of `import pandas._testing as tm`
-            from\ pandas\._testing\ import|
-            from\ pandas\ import\ _testing\ as\ tm|
+            from\ pandas\._testing\ import
+            |from\ pandas\ import\ _testing\ as\ tm
 
             # No direct imports from conftest
-            conftest\ import|
-            import\ conftest
+            |conftest\ import
+            |import\ conftest
 
             # Check for use of pandas.testing instead of tm
-            pd\.testing\.
+            |pd\.testing\.
         types: [python]
         files: ^pandas/tests/
     -   id: incorrect-code-directives
@@ -148,9 +148,9 @@ repos:
         name: Check for outdated annotation syntax and missing error codes
         entry: |
             (?x)
-            \#\ type:\ (?!ignore)|
-            \#\ type:\s?ignore(?!\[)|
-            \)\ ->\ \"
+            \#\ type:\ (?!ignore)
+            |\#\ type:\s?ignore(?!\[)
+            |\)\ ->\ \"
         language: pygrep
         types: [python]
     -   id: np-bool
@@ -166,9 +166,9 @@ repos:
         files: ^pandas/tests/
         exclude: |
             (?x)^
-            pandas/tests/io/excel/test_writers\.py|
-            pandas/tests/io/pytables/common\.py|
-            pandas/tests/io/pytables/test_store\.py$
+            pandas/tests/io/excel/test_writers\.py
+            |pandas/tests/io/pytables/common\.py
+            |pandas/tests/io/pytables/test_store\.py$
 -   repo: https://github.com/asottile/yesqa
     rev: v1.2.2
     hooks:

asv_bench/benchmarks/series_methods.py

@@ -25,17 +25,32 @@ def time_constructor(self, data):
 
 class IsIn:
 
-    params = ["int64", "uint64", "object"]
+    params = ["int64", "uint64", "object", "Int64"]
     param_names = ["dtype"]
 
     def setup(self, dtype):
-        self.s = Series(np.random.randint(1, 10, 100000)).astype(dtype)
+        N = 10000
+        self.s = Series(np.random.randint(1, 10, N)).astype(dtype)
         self.values = [1, 2]
 
     def time_isin(self, dtypes):
         self.s.isin(self.values)
 
 
+class IsInBoolean:
+
+    params = ["boolean", "bool"]
+    param_names = ["dtype"]
+
+    def setup(self, dtype):
+        N = 10000
+        self.s = Series(np.random.randint(0, 2, N)).astype(dtype)
+        self.values = [True, False]
+
+    def time_isin(self, dtypes):
+        self.s.isin(self.values)
+
+
 class IsInDatetime64:
     def setup(self):
         dti = date_range(
@@ -59,21 +74,27 @@ def time_isin_empty(self):
 
 
 class IsInFloat64:
-    def setup(self):
-        self.small = Series([1, 2], dtype=np.float64)
-        self.many_different_values = np.arange(10 ** 6, dtype=np.float64)
-        self.few_different_values = np.zeros(10 ** 7, dtype=np.float64)
-        self.only_nans_values = np.full(10 ** 7, np.nan, dtype=np.float64)
 
-    def time_isin_many_different(self):
+    params = [np.float64, "Float64"]
+    param_names = ["dtype"]
+
+    def setup(self, dtype):
+        N_many = 10 ** 5
+        N_few = 10 ** 6
+        self.small = Series([1, 2], dtype=dtype)
+        self.many_different_values = np.arange(N_many, dtype=np.float64)
+        self.few_different_values = np.zeros(N_few, dtype=np.float64)
+        self.only_nans_values = np.full(N_few, np.nan, dtype=np.float64)
+
+    def time_isin_many_different(self, dtypes):
         # runtime is dominated by creation of the lookup-table
         self.small.isin(self.many_different_values)
 
-    def time_isin_few_different(self):
+    def time_isin_few_different(self, dtypes):
         # runtime is dominated by creation of the lookup-table
         self.small.isin(self.few_different_values)
 
-    def time_isin_nan_values(self):
+    def time_isin_nan_values(self, dtypes):
         # runtime is dominated by creation of the lookup-table
         self.small.isin(self.few_different_values)
 
@@ -114,7 +135,7 @@ def time_isin_long_series_long_values_floats(self):
 
 class IsInLongSeriesLookUpDominates:
     params = [
-        ["int64", "int32", "float64", "float32", "object"],
+        ["int64", "int32", "float64", "float32", "object", "Int64", "Float64"],
         [5, 1000],
         ["random_hits", "random_misses", "monotone_hits", "monotone_misses"],
     ]
@@ -141,7 +162,7 @@ def time_isin(self, dtypes, MaxNumber, series_type):
 
 class IsInLongSeriesValuesDominate:
     params = [
-        ["int64", "int32", "float64", "float32", "object"],
+        ["int64", "int32", "float64", "float32", "object", "Int64", "Float64"],
         ["random", "monotone"],
     ]
     param_names = ["dtype", "series_type"]

doc/source/reference/extensions.rst

@@ -48,6 +48,7 @@ objects.
       api.extensions.ExtensionArray.equals
       api.extensions.ExtensionArray.factorize
       api.extensions.ExtensionArray.fillna
+      api.extensions.ExtensionArray.isin
       api.extensions.ExtensionArray.isna
       api.extensions.ExtensionArray.ravel
       api.extensions.ExtensionArray.repeat

doc/source/whatsnew/index.rst

@@ -24,6 +24,7 @@ Version 1.2
 .. toctree::
    :maxdepth: 2
 
+   v1.2.2
    v1.2.1
    v1.2.0
 

doc/source/whatsnew/v1.2.0.rst

@@ -286,6 +286,8 @@ Other enhancements
 - Added methods :meth:`IntegerArray.prod`, :meth:`IntegerArray.min`, and :meth:`IntegerArray.max` (:issue:`33790`)
 - Calling a NumPy ufunc on a ``DataFrame`` with extension types now preserves the extension types when possible (:issue:`23743`)
 - Calling a binary-input NumPy ufunc on multiple ``DataFrame`` objects now aligns, matching the behavior of binary operations and ufuncs on ``Series`` (:issue:`23743`).
+  This change has been reverted in pandas 1.2.1, and the behaviour to not align DataFrames
+  is deprecated instead, see the :ref:`the 1.2.1 release notes <whatsnew_121.ufunc_deprecation>`.
 - Where possible :meth:`RangeIndex.difference` and :meth:`RangeIndex.symmetric_difference` will return :class:`RangeIndex` instead of :class:`Int64Index` (:issue:`36564`)
 - :meth:`DataFrame.to_parquet` now supports :class:`MultiIndex` for columns in parquet format (:issue:`34777`)
 - :func:`read_parquet` gained a ``use_nullable_dtypes=True`` option to use nullable dtypes that use ``pd.NA`` as missing value indicator where possible for the resulting DataFrame (default is ``False``, and only applicable for ``engine="pyarrow"``) (:issue:`31242`)
@@ -536,6 +538,14 @@ Deprecations
 - The ``inplace`` parameter of :meth:`Categorical.remove_unused_categories` is deprecated and will be removed in a future version (:issue:`37643`)
 - The ``null_counts`` parameter of :meth:`DataFrame.info` is deprecated and replaced by ``show_counts``. It will be removed in a future version (:issue:`37999`)
 
+**Calling NumPy ufuncs on non-aligned DataFrames**
+
+Calling NumPy ufuncs on non-aligned DataFrames changed behaviour in pandas
+1.2.0 (to align the inputs before calling the ufunc), but this change is
+reverted in pandas 1.2.1. The behaviour to not align is now deprecated instead,
+see the :ref:`the 1.2.1 release notes <whatsnew_121.ufunc_deprecation>` for
+more details.
+
 .. ---------------------------------------------------------------------------
 
 

doc/source/whatsnew/v1.2.1.rst

@@ -1,6 +1,6 @@
 .. _whatsnew_121:
 
-What's new in 1.2.1 (January 18, 2021)
+What's new in 1.2.1 (January 20, 2021)
 --------------------------------------
 
 These are the changes in pandas 1.2.1. See :ref:`release` for a full changelog
@@ -42,6 +42,79 @@ As a result, bugs reported as fixed in pandas 1.2.0 related to inconsistent tick
 
 .. ---------------------------------------------------------------------------
 
+.. _whatsnew_121.ufunc_deprecation:
+
+Calling NumPy ufuncs on non-aligned DataFrames
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Before pandas 1.2.0, calling a NumPy ufunc on non-aligned DataFrames (or
+DataFrame / Series combination) would ignore the indices, only match
+the inputs by shape, and use the index/columns of the first DataFrame for
+the result:
+
+.. code-block:: python
+
+    >>> df1 = pd.DataFrame({"a": [1, 2], "b": [3, 4]}, index=[0, 1])
+    ... df2 = pd.DataFrame({"a": [1, 2], "b": [3, 4]}, index=[1, 2])
+    >>> df1
+       a  b
+    0  1  3
+    1  2  4
+    >>> df2
+       a  b
+    1  1  3
+    2  2  4
+
+    >>> np.add(df1, df2)
+       a  b
+    0  2  6
+    1  4  8
+
+This contrasts with how other pandas operations work, which first align
+the inputs:
+
+.. code-block:: python
+
+    >>> df1 + df2
+         a    b
+    0  NaN  NaN
+    1  3.0  7.0
+    2  NaN  NaN
+
+In pandas 1.2.0, we refactored how NumPy ufuncs are called on DataFrames, and
+this started to align the inputs first (:issue:`39184`), as happens in other
+pandas operations and as it happens for ufuncs called on Series objects.
+
+For pandas 1.2.1, we restored the previous behaviour to avoid a breaking
+change, but the above example of ``np.add(df1, df2)`` with non-aligned inputs
+will now to raise a warning, and a future pandas 2.0 release will start
+aligning the inputs first (:issue:`39184`). Calling a NumPy ufunc on Series
+objects (eg ``np.add(s1, s2)``) already aligns and continues to do so.
+
+To avoid the warning and keep the current behaviour of ignoring the indices,
+convert one of the arguments to a NumPy array:
+
+.. code-block:: python
+
+    >>> np.add(df1, np.asarray(df2))
+       a  b
+    0  2  6
+    1  4  8
+
+To obtain the future behaviour and silence the warning, you can align manually
+before passing the arguments to the ufunc:
+
+.. code-block:: python
+
+    >>> df1, df2 = df1.align(df2)
+    >>> np.add(df1, df2)
+         a    b
+    0  NaN  NaN
+    1  3.0  7.0
+    2  NaN  NaN
+
+.. ---------------------------------------------------------------------------
+
 .. _whatsnew_121.bug_fixes:
 
 Bug fixes
@@ -71,4 +144,4 @@ Other
 Contributors
 ~~~~~~~~~~~~
 
-.. contributors:: v1.2.0..v1.2.1|HEAD
+.. contributors:: v1.2.0..v1.2.1

doc/source/whatsnew/v1.2.2.rst

@@ -0,0 +1,47 @@
+.. _whatsnew_122:
+
+What's new in 1.2.2 (February ??, 2021)
+---------------------------------------
+
+These are the changes in pandas 1.2.2. See :ref:`release` for a full changelog
+including other versions of pandas.
+
+{{ header }}
+
+.. ---------------------------------------------------------------------------
+
+.. _whatsnew_122.regressions:
+
+Fixed regressions
+~~~~~~~~~~~~~~~~~
+-
+-
+
+.. ---------------------------------------------------------------------------
+
+.. _whatsnew_122.bug_fixes:
+
+Bug fixes
+~~~~~~~~~
+
+-
+-
+
+.. ---------------------------------------------------------------------------
+
+.. _whatsnew_122.other:
+
+Other
+~~~~~
+
+-
+-
+
+.. ---------------------------------------------------------------------------
+
+.. _whatsnew_122.contributors:
+
+Contributors
+~~~~~~~~~~~~
+
+.. contributors:: v1.2.1..v1.2.2|HEAD

doc/source/whatsnew/v1.3.0.rst

@@ -206,7 +206,8 @@ Performance improvements
 ~~~~~~~~~~~~~~~~~~~~~~~~
 - Performance improvement in :meth:`IntervalIndex.isin` (:issue:`38353`)
 - Performance improvement in :meth:`Series.mean` for nullable data types (:issue:`34814`)
--
+- Performance improvement in :meth:`Series.isin` for nullable data types (:issue:`38340`)
+- Performance improvement in :meth:`DataFrame.corr` for method=kendall (:issue:`28329`)
 
 .. ---------------------------------------------------------------------------
 
@@ -243,7 +244,7 @@ Timedelta
 
 Timezones
 ^^^^^^^^^
-
+- Bug in different ``tzinfo`` objects representing UTC not being treated as equivalent (:issue:`39216`)
 -
 -
 
@@ -271,7 +272,7 @@ Interval
 ^^^^^^^^
 - Bug in :meth:`IntervalIndex.intersection` and :meth:`IntervalIndex.symmetric_difference` always returning object-dtype when operating with :class:`CategoricalIndex` (:issue:`38653`, :issue:`38741`)
 - Bug in :meth:`IntervalIndex.intersection` returning duplicates when at least one of both Indexes has duplicates which are present in the other (:issue:`38743`)
--
+- :meth:`IntervalIndex.union`, :meth:`IntervalIndex.intersection`, :meth:`IntervalIndex.difference`, and :meth:`IntervalIndex.symmetric_difference` now cast to the appropriate dtype instead of raising ``TypeError`` when operating with another :class:`IntervalIndex` with incompatible dtype (:issue:`39267`)
 
 Indexing
 ^^^^^^^^
@@ -323,7 +324,7 @@ I/O
 
 Period
 ^^^^^^
-
+- Comparisons of :class:`Period` objects or :class:`Index`, :class:`Series`, or :class:`DataFrame` with mismatched ``PeriodDtype`` now behave like other mismatched-type comparisons, returning ``False`` for equals, ``True`` for not-equal, and raising ``TypeError`` for inequality checks (:issue:`??`)
 -
 -
 
@@ -342,6 +343,7 @@ Groupby/resample/rolling
 - Fixed bug in :meth:`DataFrameGroupBy.sum` and :meth:`SeriesGroupBy.sum` causing loss of precision through using Kahan summation (:issue:`38778`)
 - Fixed bug in :meth:`DataFrameGroupBy.cumsum`, :meth:`SeriesGroupBy.cumsum`, :meth:`DataFrameGroupBy.mean` and :meth:`SeriesGroupBy.mean` causing loss of precision through using Kahan summation (:issue:`38934`)
 - Bug in :meth:`.Resampler.aggregate` and :meth:`DataFrame.transform` raising ``TypeError`` instead of ``SpecificationError`` when missing keys had mixed dtypes (:issue:`39025`)
+- Bug in :meth:`.DataFrameGroupBy.idxmin` and :meth:`.DataFrameGroupBy.idxmax` with ``ExtensionDtype`` columns (:issue:`38733`)
 
 Reshaping
 ^^^^^^^^^
@@ -350,6 +352,7 @@ Reshaping
 - Bug in :func:`join` over :class:`MultiIndex` returned wrong result, when one of both indexes had only one level (:issue:`36909`)
 - :meth:`merge_asof` raises ``ValueError`` instead of cryptic ``TypeError`` in case of non-numerical merge columns (:issue:`29130`)
 - Bug in :meth:`DataFrame.join` not assigning values correctly when having :class:`MultiIndex` where at least one dimension is from dtype ``Categorical`` with non-alphabetically sorted categories (:issue:`38502`)
+- Bug in :meth:`DataFrame.apply` would give incorrect results when used with a string argument and ``axis=1`` when the axis argument was not supported and now raises a ``ValueError`` instead (:issue:`39211`)
 -
 
 Sparse