Merge https://github.com/pandas-dev/pandas

Author: OliEfr

asv_bench/benchmarks/dtypes.py

@@ -2,9 +2,10 @@
 
 import numpy as np
 
+import pandas as pd
 from pandas import DataFrame
 import pandas._testing as tm
-from pandas.api.types import pandas_dtype
+from pandas.api.types import is_extension_array_dtype, pandas_dtype
 
 from .pandas_vb_common import (
     datetime_dtypes,
@@ -119,4 +120,16 @@ def time_select_dtype_string_exclude(self, dtype):
         self.df_string.select_dtypes(exclude=dtype)
 
 
+class CheckDtypes:
+    def setup(self):
+        self.ext_dtype = pd.Int64Dtype()
+        self.np_dtype = np.dtype("int64")
+
+    def time_is_extension_array_dtype_true(self):
+        is_extension_array_dtype(self.ext_dtype)
+
+    def time_is_extension_array_dtype_false(self):
+        is_extension_array_dtype(self.np_dtype)
+
+
 from .pandas_vb_common import setup  # noqa: F401 isort:skip

environment.yml

@@ -114,4 +114,4 @@ dependencies:
   - natsort  # DataFrame.sort_values
   - pip:
     - git+https://github.com/pandas-dev/pydata-sphinx-theme.git@2488b7defbd3d753dd5fcfc890fc4a7e79d25103
-    - git+https://github.com/numpy/numpydoc
+    - numpydoc < 1.2  # 2021-02-09 1.2dev breaking CI

pandas/core/arrays/categorical.py

@@ -54,7 +54,7 @@
 )
 from pandas.core.dtypes.dtypes import CategoricalDtype
 from pandas.core.dtypes.generic import ABCIndex, ABCSeries
-from pandas.core.dtypes.missing import is_valid_nat_for_dtype, isna, notna
+from pandas.core.dtypes.missing import is_valid_na_for_dtype, isna, notna
 
 from pandas.core import ops
 from pandas.core.accessor import PandasDelegate, delegate_names
@@ -1284,7 +1284,7 @@ def _validate_fill_value(self, fill_value):
         TypeError
         """
 
-        if is_valid_nat_for_dtype(fill_value, self.categories.dtype):
+        if is_valid_na_for_dtype(fill_value, self.categories.dtype):
             fill_value = -1
         elif fill_value in self.categories:
             fill_value = self._unbox_scalar(fill_value)
@@ -1779,7 +1779,7 @@ def __contains__(self, key) -> bool:
         Returns True if `key` is in this Categorical.
         """
         # if key is a NaN, check if any NaN is in self.
-        if is_valid_nat_for_dtype(key, self.categories.dtype):
+        if is_valid_na_for_dtype(key, self.categories.dtype):
             return self.isna().any()
 
         return contains(self, key, container=self._codes)

pandas/core/arrays/datetimelike.py

@@ -57,7 +57,7 @@
     is_unsigned_integer_dtype,
     pandas_dtype,
 )
-from pandas.core.dtypes.missing import is_valid_nat_for_dtype, isna
+from pandas.core.dtypes.missing import is_valid_na_for_dtype, isna
 
 from pandas.core import nanops, ops
 from pandas.core.algorithms import checked_add_with_arr, isin, unique1d
@@ -493,7 +493,7 @@ def _validate_fill_value(self, fill_value):
 
     def _validate_shift_value(self, fill_value):
         # TODO(2.0): once this deprecation is enforced, use _validate_fill_value
-        if is_valid_nat_for_dtype(fill_value, self.dtype):
+        if is_valid_na_for_dtype(fill_value, self.dtype):
             fill_value = NaT
         elif isinstance(fill_value, self._recognized_scalars):
             # pandas\core\arrays\datetimelike.py:746: error: Too many arguments
@@ -557,7 +557,7 @@ def _validate_scalar(
                 msg = self._validation_error_message(value, allow_listlike)
                 raise TypeError(msg) from err
 
-        elif is_valid_nat_for_dtype(value, self.dtype):
+        elif is_valid_na_for_dtype(value, self.dtype):
             # GH#18295
             value = NaT
 

pandas/core/arrays/interval.py

@@ -43,7 +43,7 @@
     ABCPeriodIndex,
     ABCSeries,
 )
-from pandas.core.dtypes.missing import is_valid_nat_for_dtype, isna, notna
+from pandas.core.dtypes.missing import is_valid_na_for_dtype, isna, notna
 
 from pandas.core.algorithms import isin, take, value_counts
 from pandas.core.arrays.base import ExtensionArray, _extension_array_shared_docs
@@ -979,7 +979,7 @@ def _validate_scalar(self, value):
         if isinstance(value, Interval):
             self._check_closed_matches(value, name="value")
             left, right = value.left, value.right
-        elif is_valid_nat_for_dtype(value, self.left.dtype):
+        elif is_valid_na_for_dtype(value, self.left.dtype):
             # GH#18295
             left = right = value
         else:
@@ -994,7 +994,7 @@ def _validate_fill_value(self, value):
     def _validate_setitem_value(self, value):
         needs_float_conversion = False
 
-        if is_valid_nat_for_dtype(value, self.left.dtype):
+        if is_valid_na_for_dtype(value, self.left.dtype):
             # na value: need special casing to set directly on numpy arrays
             if is_integer_dtype(self.dtype.subtype):
                 # can't set NaN on a numpy integer array

pandas/core/dtypes/cast.py

@@ -87,7 +87,7 @@
     ABCSeries,
 )
 from pandas.core.dtypes.inference import is_list_like
-from pandas.core.dtypes.missing import is_valid_nat_for_dtype, isna, notna
+from pandas.core.dtypes.missing import is_valid_na_for_dtype, isna, notna
 
 if TYPE_CHECKING:
     from pandas import Series
@@ -159,7 +159,7 @@ def maybe_unbox_datetimelike(value: Scalar, dtype: DtypeObj) -> Scalar:
     -----
     Caller is responsible for checking dtype.kind in ["m", "M"]
     """
-    if is_valid_nat_for_dtype(value, dtype):
+    if is_valid_na_for_dtype(value, dtype):
         # GH#36541: can't fill array directly with pd.NaT
         # > np.empty(10, dtype="datetime64[64]").fill(pd.NaT)
         # ValueError: cannot convert float NaN to integer
@@ -535,7 +535,7 @@ def maybe_promote(dtype, fill_value=np.nan):
             dtype = np.dtype(np.object_)
         elif is_integer(fill_value) or (is_float(fill_value) and not isna(fill_value)):
             dtype = np.dtype(np.object_)
-        elif is_valid_nat_for_dtype(fill_value, dtype):
+        elif is_valid_na_for_dtype(fill_value, dtype):
             # e.g. pd.NA, which is not accepted by Timestamp constructor
             fill_value = np.datetime64("NaT", "ns")
         else:
@@ -551,7 +551,7 @@ def maybe_promote(dtype, fill_value=np.nan):
         ):
             # TODO: What about str that can be a timedelta?
             dtype = np.dtype(np.object_)
-        elif is_valid_nat_for_dtype(fill_value, dtype):
+        elif is_valid_na_for_dtype(fill_value, dtype):
             # e.g pd.NA, which is not accepted by the  Timedelta constructor
             fill_value = np.timedelta64("NaT", "ns")
         else:

pandas/core/dtypes/common.py

@@ -1526,7 +1526,12 @@ def is_extension_array_dtype(arr_or_dtype) -> bool:
     False
     """
     dtype = getattr(arr_or_dtype, "dtype", arr_or_dtype)
-    return isinstance(dtype, ExtensionDtype) or registry.find(dtype) is not None
+    if isinstance(dtype, ExtensionDtype):
+        return True
+    elif isinstance(dtype, np.dtype):
+        return False
+    else:
+        return registry.find(dtype) is not None
 
 
 def is_ea_or_datetimelike_dtype(dtype: Optional[DtypeObj]) -> bool:

pandas/core/dtypes/missing.py

@@ -29,7 +29,6 @@
     is_string_dtype,
     is_string_like_dtype,
     needs_i8_conversion,
-    pandas_dtype,
 )
 from pandas.core.dtypes.generic import (
     ABCDataFrame,
@@ -535,7 +534,7 @@ def maybe_fill(arr, fill_value=np.nan):
     return arr
 
 
-def na_value_for_dtype(dtype, compat: bool = True):
+def na_value_for_dtype(dtype: DtypeObj, compat: bool = True):
     """
     Return a dtype compat na value
 
@@ -561,7 +560,6 @@ def na_value_for_dtype(dtype, compat: bool = True):
     >>> na_value_for_dtype(np.dtype('datetime64[ns]'))
     numpy.datetime64('NaT')
     """
-    dtype = pandas_dtype(dtype)
 
     if is_extension_array_dtype(dtype):
         return dtype.na_value
@@ -590,7 +588,7 @@ def remove_na_arraylike(arr):
         return arr[notna(np.asarray(arr))]
 
 
-def is_valid_nat_for_dtype(obj, dtype: DtypeObj) -> bool:
+def is_valid_na_for_dtype(obj, dtype: DtypeObj) -> bool:
     """
     isna check that excludes incompatible dtypes
 

pandas/core/frame.py

@@ -3272,16 +3272,13 @@ def _set_item_frame_value(self, key, value: DataFrame) -> None:
 
         # now align rows
         value = _reindex_for_setitem(value, self.index)
-        value = value.T
         self._set_item_mgr(key, value)
 
     def _iset_item_mgr(self, loc: int, value) -> None:
         self._mgr.iset(loc, value)
         self._clear_item_cache()
 
     def _set_item_mgr(self, key, value):
-        value = _maybe_atleast_2d(value)
-
         try:
             loc = self._info_axis.get_loc(key)
         except KeyError:
@@ -3298,7 +3295,6 @@ def _set_item_mgr(self, key, value):
 
     def _iset_item(self, loc: int, value):
         value = self._sanitize_column(value)
-        value = _maybe_atleast_2d(value)
         self._iset_item_mgr(loc, value)
 
         # check if we are modifying a copy
@@ -3328,7 +3324,7 @@ def _set_item(self, key, value):
             if not self.columns.is_unique or isinstance(self.columns, MultiIndex):
                 existing_piece = self[key]
                 if isinstance(existing_piece, DataFrame):
-                    value = np.tile(value, (len(existing_piece.columns), 1))
+                    value = np.tile(value, (len(existing_piece.columns), 1)).T
 
         self._set_item_mgr(key, value)
 
@@ -3889,7 +3885,6 @@ def insert(self, loc, column, value, allow_duplicates: bool = False) -> None:
                 "'self.flags.allows_duplicate_labels' is False."
             )
         value = self._sanitize_column(value)
-        value = _maybe_atleast_2d(value)
         self._mgr.insert(loc, column, value, allow_duplicates=allow_duplicates)
 
     def assign(self, **kwargs) -> DataFrame:
@@ -3994,8 +3989,6 @@ def _sanitize_column(self, value):
                     value = maybe_convert_platform(value)
                 else:
                     value = com.asarray_tuplesafe(value)
-            elif value.ndim == 2:
-                value = value.copy().T
             elif isinstance(value, Index):
                 value = value.copy(deep=True)
             else:

pandas/core/indexes/base.py

@@ -88,7 +88,7 @@
     ABCTimedeltaIndex,
 )
 from pandas.core.dtypes.inference import is_dict_like
-from pandas.core.dtypes.missing import array_equivalent, is_valid_nat_for_dtype, isna
+from pandas.core.dtypes.missing import array_equivalent, is_valid_na_for_dtype, isna
 
 from pandas.core import missing, ops
 from pandas.core.accessor import CachedAccessor
@@ -2620,36 +2620,15 @@ def duplicated(self, keep="first"):
             return np.zeros(len(self), dtype=bool)
         return super().duplicated(keep=keep)
 
-    def _get_unique_index(self, dropna: bool = False):
+    def _get_unique_index(self: _IndexT) -> _IndexT:
         """
         Returns an index containing unique values.
 
-        Parameters
-        ----------
-        dropna : bool, default False
-            If True, NaN values are dropped.
-
         Returns
         -------
-        uniques : index
+        Index
         """
-        if self.is_unique and not dropna:
-            return self
-
-        if not self.is_unique:
-            values = self.unique()
-            if not isinstance(self, ABCMultiIndex):
-                # extract an array to pass to _shallow_copy
-                values = values._data
-        else:
-            values = self._values
-
-        if dropna and not isinstance(self, ABCMultiIndex):
-            # isna not defined for MultiIndex
-            if self.hasnans:
-                values = values[~isna(values)]
-
-        return self._shallow_copy(values)
+        return self.unique()
 
     # --------------------------------------------------------------------
     # Arithmetic & Logical Methods
@@ -5216,7 +5195,7 @@ def _find_common_type_compat(self, target) -> DtypeObj:
         Implementation of find_common_type that adjusts for Index-specific
         special cases.
         """
-        if is_interval_dtype(self.dtype) and is_valid_nat_for_dtype(target, self.dtype):
+        if is_interval_dtype(self.dtype) and is_valid_na_for_dtype(target, self.dtype):
             # e.g. setting NA value into IntervalArray[int64]
             self = cast("IntervalIndex", self)
             return IntervalDtype(np.float64, closed=self.closed)
@@ -5770,7 +5749,7 @@ def insert(self, loc: int, item):
         # Note: this method is overridden by all ExtensionIndex subclasses,
         #  so self is never backed by an EA.
         item = lib.item_from_zerodim(item)
-        if is_valid_nat_for_dtype(item, self.dtype) and self.dtype != object:
+        if is_valid_na_for_dtype(item, self.dtype) and self.dtype != object:
             item = self._na_value
 
         try:

pandas/core/indexes/category.py

@@ -15,7 +15,7 @@
     is_categorical_dtype,
     is_scalar,
 )
-from pandas.core.dtypes.missing import is_valid_nat_for_dtype, isna, notna
+from pandas.core.dtypes.missing import is_valid_na_for_dtype, isna, notna
 
 from pandas.core import accessor
 from pandas.core.arrays.categorical import Categorical, contains
@@ -348,7 +348,7 @@ def inferred_type(self) -> str:
     @doc(Index.__contains__)
     def __contains__(self, key: Any) -> bool:
         # if key is a NaN, check if any NaN is in self.
-        if is_valid_nat_for_dtype(key, self.categories.dtype):
+        if is_valid_na_for_dtype(key, self.categories.dtype):
             return self.hasnans
 
         return contains(self, key, container=self._engine)

pandas/core/indexes/datetimes.py

@@ -26,7 +26,7 @@
     is_datetime64tz_dtype,
     is_scalar,
 )
-from pandas.core.dtypes.missing import is_valid_nat_for_dtype
+from pandas.core.dtypes.missing import is_valid_na_for_dtype
 
 from pandas.core.arrays.datetimes import DatetimeArray, tz_to_dtype
 import pandas.core.common as com
@@ -636,7 +636,7 @@ def get_loc(self, key, method=None, tolerance=None):
             raise InvalidIndexError(key)
 
         orig_key = key
-        if is_valid_nat_for_dtype(key, self.dtype):
+        if is_valid_na_for_dtype(key, self.dtype):
             key = NaT
 
         if isinstance(key, self._data._recognized_scalars):

pandas/core/indexes/extension.py

@@ -263,13 +263,11 @@ def insert(self, loc: int, item):
         # ExtensionIndex subclasses must override Index.insert
         raise AbstractMethodError(self)
 
-    def _get_unique_index(self, dropna=False):
-        if self.is_unique and not dropna:
+    def _get_unique_index(self):
+        if self.is_unique:
             return self
 
         result = self._data.unique()
-        if dropna and self.hasnans:
-            result = result[~result.isna()]
         return self._shallow_copy(result)
 
     @doc(Index.map)

pandas/core/internals/array_manager.py

@@ -20,7 +20,7 @@
 )
 from pandas.core.dtypes.dtypes import ExtensionDtype, PandasDtype
 from pandas.core.dtypes.generic import ABCDataFrame, ABCSeries
-from pandas.core.dtypes.missing import isna
+from pandas.core.dtypes.missing import array_equals, isna
 
 import pandas.core.algorithms as algos
 from pandas.core.arrays import ExtensionArray
@@ -829,9 +829,16 @@ def _make_na_array(self, fill_value=None):
         values.fill(fill_value)
         return values
 
-    def equals(self, other: object) -> bool:
-        # TODO
-        raise NotImplementedError
+    def _equal_values(self, other) -> bool:
+        """
+        Used in .equals defined in base class. Only check the column values
+        assuming shape and indexes have already been checked.
+        """
+        for left, right in zip(self.arrays, other.arrays):
+            if not array_equals(left, right):
+                return False
+        else:
+            return True
 
     def unstack(self, unstacker, fill_value) -> ArrayManager:
         """