Merge branch 'master' into nui-regression

Author: alexhlim

pandas/core/generic.py

@@ -8858,7 +8858,7 @@ def _where(
                     elif len(cond[icond]) == len(other):
 
                         # try to not change dtype at first
-                        new_other = np.asarray(self)
+                        new_other = self._values
                         new_other = new_other.copy()
                         new_other[icond] = other
                         other = new_other

pandas/core/indexes/base.py

@@ -2906,8 +2906,13 @@ def difference(self, other, sort=None):
         other, result_name = self._convert_can_do_setop(other)
 
         if self.equals(other):
+            # Note: we do not (yet) sort even if sort=None GH#24959
             return self[:0].rename(result_name)
 
+        if len(other) == 0:
+            # Note: we do not (yet) sort even if sort=None GH#24959
+            return self.rename(result_name)
+
         result = self._difference(other, sort=sort)
         return self._wrap_setop_result(other, result)
 

pandas/core/indexes/multi.py

@@ -3676,42 +3676,9 @@ def _intersection(self, other, sort=False):
                 zip(*uniq_tuples), sortorder=0, names=result_names
             )
 
-    def difference(self, other, sort=None):
-        """
-        Compute set difference of two MultiIndex objects
-
-        Parameters
-        ----------
-        other : MultiIndex
-        sort : False or None, default None
-            Sort the resulting MultiIndex if possible
-
-            .. versionadded:: 0.24.0
-
-            .. versionchanged:: 0.24.1
-
-               Changed the default value from ``True`` to ``None``
-               (without change in behaviour).
-
-        Returns
-        -------
-        diff : MultiIndex
-        """
-        self._validate_sort_keyword(sort)
-        self._assert_can_do_setop(other)
+    def _difference(self, other, sort):
         other, result_names = self._convert_can_do_setop(other)
 
-        if len(other) == 0:
-            return self.rename(result_names)
-
-        if self.equals(other):
-            return MultiIndex(
-                levels=self.levels,
-                codes=[[]] * self.nlevels,
-                names=result_names,
-                verify_integrity=False,
-            )
-
         this = self._get_unique_index()
 
         indexer = this.get_indexer(other)

pandas/core/indexes/period.py

@@ -635,16 +635,6 @@ def _setop(self, other, sort, opname: str):
     def _intersection(self, other, sort=False):
         return self._setop(other, sort, opname="intersection")
 
-    def difference(self, other, sort=None):
-        self._validate_sort_keyword(sort)
-        self._assert_can_do_setop(other)
-        other, result_name = self._convert_can_do_setop(other)
-
-        if self.equals(other):
-            return self[:0].rename(result_name)
-
-        return self._difference(other, sort=sort)
-
     def _difference(self, other, sort):
 
         if is_object_dtype(other):

pandas/core/internals/blocks.py

@@ -1,4 +1,3 @@
-from datetime import timedelta
 import inspect
 import re
 from typing import TYPE_CHECKING, Any, List, Optional, Type, Union, cast
@@ -8,7 +7,6 @@
 
 from pandas._libs import (
     Interval,
-    NaT,
     Period,
     Timestamp,
     algos as libalgos,
@@ -86,6 +84,7 @@
 
 if TYPE_CHECKING:
     from pandas import Index
+    from pandas.core.arrays._mixins import NDArrayBackedExtensionArray
 
 
 class Block(PandasObject):
@@ -919,7 +918,11 @@ def setitem(self, indexer, value):
         if self._can_hold_element(value):
             # We only get here for non-Extension Blocks, so _try_coerce_args
             #  is only relevant for DatetimeBlock and TimedeltaBlock
-            if lib.is_scalar(value):
+            if self.dtype.kind in ["m", "M"]:
+                arr = self.array_values().T
+                arr[indexer] = value
+                return self
+            elif lib.is_scalar(value):
                 value = convert_scalar_for_putitemlike(value, values.dtype)
 
         else:
@@ -1029,9 +1032,7 @@ def _putmask_simple(self, mask: np.ndarray, value: Any):
             # GH#37833 np.putmask is more performant than __setitem__
             np.putmask(values, mask, value)
 
-    def putmask(
-        self, mask, new, inplace: bool = False, axis: int = 0, transpose: bool = False
-    ) -> List["Block"]:
+    def putmask(self, mask, new, axis: int = 0) -> List["Block"]:
         """
         putmask the data to the block; it is possible that we may create a
         new dtype of block
@@ -1042,16 +1043,13 @@ def putmask(
         ----------
         mask : np.ndarray[bool], SparseArray[bool], or BooleanArray
         new : a ndarray/object
-        inplace : bool, default False
-            Perform inplace modification.
         axis : int
-        transpose : bool, default False
-            Set to True if self is stored with axes reversed.
 
         Returns
         -------
         List[Block]
         """
+        transpose = self.ndim == 2
         mask = _extract_bool_array(mask)
         assert not isinstance(new, (ABCIndex, ABCSeries, ABCDataFrame))
 
@@ -1064,6 +1062,17 @@ def putmask(
         if self._can_hold_element(new):
             # We only get here for non-Extension Blocks, so _try_coerce_args
             #  is only relevant for DatetimeBlock and TimedeltaBlock
+            if self.dtype.kind in ["m", "M"]:
+                blk = self
+                if not inplace:
+                    blk = self.copy()
+                arr = blk.array_values()
+                arr = cast("NDArrayBackedExtensionArray", arr)
+                if transpose:
+                    arr = arr.T
+                arr.putmask(mask, new)
+                return [blk]
+
             if lib.is_scalar(new):
                 new = convert_scalar_for_putitemlike(new, self.values.dtype)
 
@@ -1077,8 +1086,6 @@ def putmask(
                     new = np.repeat(new, new_values.shape[-1]).reshape(self.shape)
                 new = new.astype(new_values.dtype)
 
-            if new_values is self.values and not inplace:
-                new_values = new_values.copy()
             # we require exact matches between the len of the
             # values we are setting (or is compat). np.putmask
             # doesn't check this and will simply truncate / pad
@@ -1099,6 +1106,7 @@ def putmask(
                     # `np.place` on the other hand uses the ``new`` values at it is
                     # to place in the masked locations of ``new_values``
                     np.place(new_values, mask, new)
+                    # i.e. new_values[mask] = new
                 elif mask.shape[-1] == len(new) or len(new) == 1:
                     np.putmask(new_values, mask, new)
                 else:
@@ -1143,18 +1151,10 @@ def f(mask, val, idx):
                 nv = _putmask_smart(val, mask, n)
                 return nv
 
-            new_blocks = self.split_and_operate(mask, f, inplace)
+            new_blocks = self.split_and_operate(mask, f, True)
             return new_blocks
 
-        if inplace:
-            return [self]
-
-        if transpose:
-            if new_values is None:
-                new_values = self.values if inplace else self.values.copy()
-            new_values = new_values.T
-
-        return [self.make_block(new_values)]
+        return [self]
 
     def coerce_to_target_dtype(self, other):
         """
@@ -1697,17 +1697,13 @@ def set_inplace(self, locs, values):
         assert locs.tolist() == [0]
         self.values = values
 
-    def putmask(
-        self, mask, new, inplace: bool = False, axis: int = 0, transpose: bool = False
-    ) -> List["Block"]:
+    def putmask(self, mask, new, axis: int = 0) -> List["Block"]:
         """
         See Block.putmask.__doc__
         """
-        inplace = validate_bool_kwarg(inplace, "inplace")
-
         mask = _extract_bool_array(mask)
 
-        new_values = self.values if inplace else self.values.copy()
+        new_values = self.values
 
         if isinstance(new, (np.ndarray, ExtensionArray)) and len(new) == len(mask):
             new = new[mask]
@@ -2376,16 +2372,6 @@ def _maybe_coerce_values(self, values):
     def _holder(self):
         return TimedeltaArray
 
-    def _can_hold_element(self, element: Any) -> bool:
-        tipo = maybe_infer_dtype_type(element)
-        if tipo is not None:
-            return issubclass(tipo.type, np.timedelta64)
-        elif element is NaT:
-            return True
-        elif isinstance(element, (timedelta, np.timedelta64)):
-            return True
-        return is_valid_nat_for_dtype(element, self.dtype)
-
     def fillna(self, value, **kwargs):
         # TODO(EA2D): if we operated on array_values, TDA.fillna would handle
         #  raising here.

pandas/core/internals/managers.py

@@ -565,7 +565,6 @@ def setitem(self, indexer, value) -> "BlockManager":
         return self.apply("setitem", indexer=indexer, value=value)
 
     def putmask(self, mask, new, align: bool = True, axis: int = 0):
-        transpose = self.ndim == 2
 
         if align:
             align_keys = ["new", "mask"]
@@ -578,9 +577,7 @@ def putmask(self, mask, new, align: bool = True, axis: int = 0):
             align_keys=align_keys,
             mask=mask,
             new=new,
-            inplace=True,
             axis=axis,
-            transpose=transpose,
         )
 
     def diff(self, n: int, axis: int) -> "BlockManager":

pandas/tests/indexes/datetimes/test_setops.py

@@ -326,7 +326,8 @@ def test_difference(self, tz, sort):
             (rng3, other3, expected3),
         ]:
             result_diff = rng.difference(other, sort)
-            if sort is None:
+            if sort is None and len(other):
+                # We dont sort (yet?) when empty GH#24959
                 expected = expected.sort_values()
             tm.assert_index_equal(result_diff, expected)
 

pandas/tests/indexes/multi/test_setops.py

@@ -294,7 +294,9 @@ def test_intersection(idx, sort):
     # assert result.equals(tuples)
 
 
-@pytest.mark.parametrize("method", ["intersection", "union"])
+@pytest.mark.parametrize(
+    "method", ["intersection", "union", "difference", "symmetric_difference"]
+)
 def test_setop_with_categorical(idx, sort, method):
     other = idx.to_flat_index().astype("category")
     res_names = [None] * idx.nlevels

pandas/tests/indexes/period/test_setops.py

@@ -318,7 +318,8 @@ def test_difference(self, sort):
             (rng7, other7, expected7),
         ]:
             result_difference = rng.difference(other, sort=sort)
-            if sort is None:
+            if sort is None and len(other):
+                # We dont sort (yet?) when empty GH#24959
                 expected = expected.sort_values()
             tm.assert_index_equal(result_difference, expected)
 

pandas/tests/indexing/test_indexing.py

@@ -10,7 +10,7 @@
 from pandas.core.dtypes.common import is_float_dtype, is_integer_dtype
 
 import pandas as pd
-from pandas import DataFrame, Index, NaT, Series, date_range
+from pandas import DataFrame, Index, NaT, Series, date_range, offsets, timedelta_range
 import pandas._testing as tm
 from pandas.core.indexing import maybe_numeric_slice, non_reducing_slice
 from pandas.tests.indexing.common import _mklbl
@@ -970,19 +970,22 @@ class TestDatetimelikeCoercion:
     @pytest.mark.parametrize("indexer", [setitem, loc, iloc])
     def test_setitem_dt64_string_scalar(self, tz_naive_fixture, indexer):
         # dispatching _can_hold_element to underling DatetimeArray
-        # TODO(EA2D) use tz_naive_fixture once DatetimeBlock is backed by DTA
         tz = tz_naive_fixture
 
         dti = date_range("2016-01-01", periods=3, tz=tz)
         ser = Series(dti)
 
         values = ser._values
 
-        indexer(ser)[0] = "2018-01-01"
+        newval = "2018-01-01"
+        values._validate_setitem_value(newval)
+
+        indexer(ser)[0] = newval
 
         if tz is None:
             # TODO(EA2D): we can make this no-copy in tz-naive case too
             assert ser.dtype == dti.dtype
+            assert ser._values._data is values._data
         else:
             assert ser._values is values
 
@@ -993,7 +996,6 @@ def test_setitem_dt64_string_scalar(self, tz_naive_fixture, indexer):
     @pytest.mark.parametrize("indexer", [setitem, loc, iloc])
     def test_setitem_dt64_string_values(self, tz_naive_fixture, indexer, key, box):
         # dispatching _can_hold_element to underling DatetimeArray
-        # TODO(EA2D) use tz_naive_fixture once DatetimeBlock is backed by DTA
         tz = tz_naive_fixture
 
         if isinstance(key, slice) and indexer is loc:
@@ -1012,9 +1014,44 @@ def test_setitem_dt64_string_values(self, tz_naive_fixture, indexer, key, box):
         if tz is None:
             # TODO(EA2D): we can make this no-copy in tz-naive case too
             assert ser.dtype == dti.dtype
+            assert ser._values._data is values._data
         else:
             assert ser._values is values
 
+    @pytest.mark.parametrize("scalar", ["3 Days", offsets.Hour(4)])
+    @pytest.mark.parametrize("indexer", [setitem, loc, iloc])
+    def test_setitem_td64_scalar(self, indexer, scalar):
+        # dispatching _can_hold_element to underling TimedeltaArray
+        tdi = timedelta_range("1 Day", periods=3)
+        ser = Series(tdi)
+
+        values = ser._values
+        values._validate_setitem_value(scalar)
+
+        indexer(ser)[0] = scalar
+        assert ser._values._data is values._data
+
+    @pytest.mark.parametrize("box", [list, np.array, pd.array])
+    @pytest.mark.parametrize(
+        "key", [[0, 1], slice(0, 2), np.array([True, True, False])]
+    )
+    @pytest.mark.parametrize("indexer", [setitem, loc, iloc])
+    def test_setitem_td64_string_values(self, indexer, key, box):
+        # dispatching _can_hold_element to underling TimedeltaArray
+        if isinstance(key, slice) and indexer is loc:
+            key = slice(0, 1)
+
+        tdi = timedelta_range("1 Day", periods=3)
+        ser = Series(tdi)
+
+        values = ser._values
+
+        newvals = box(["10 Days", "44 hours"])
+        values._validate_setitem_value(newvals)
+
+        indexer(ser)[key] = newvals
+        assert ser._values._data is values._data
+
 
 def test_extension_array_cross_section():
     # A cross-section of a homogeneous EA should be an EA