BUG: Fix issue with datetime[ns, tz] input in Block.setitem GH32395

doc/source/whatsnew/index.rst

@@ -24,6 +24,7 @@ Version 1.0
 .. toctree::
    :maxdepth: 2
 
+   v1.0.4
    v1.0.3
    v1.0.2
    v1.0.1

doc/source/whatsnew/v1.0.3.rst

@@ -26,4 +26,4 @@ Bug fixes
 Contributors
 ~~~~~~~~~~~~
 
-.. contributors:: v1.0.2..v1.0.3|HEAD
+.. contributors:: v1.0.2..v1.0.3

doc/source/whatsnew/v1.0.4.rst

@@ -0,0 +1,28 @@
+
+.. _whatsnew_104:
+
+What's new in 1.0.4 (April 15, 2020)
+------------------------------------
+
+These are the changes in pandas 1.0.4. See :ref:`release` for a full changelog
+including other versions of pandas.
+
+{{ header }}
+
+.. ---------------------------------------------------------------------------
+
+.. _whatsnew_104.regressions:
+
+Fixed regressions
+~~~~~~~~~~~~~~~~~
+- Fixed regression in :meth:`DataFrame.loc`, :meth:`Series.loc` throwing an error when a ``datetime64[ns, tz]`` value is provided (:issue:`32395`)
+
+.. _whatsnew_104.bug_fixes:
+
+Bug fixes
+~~~~~~~~~
+
+Contributors
+~~~~~~~~~~~~
+
+.. contributors:: v1.0.3..v1.0.4|HEAD

doc/source/whatsnew/v1.1.0.rst

@@ -361,7 +361,7 @@ Indexing
 - Fix to preserve the ability to index with the "nearest" method with xarray's CFTimeIndex, an :class:`Index` subclass (`pydata/xarray#3751 <https://github.com/pydata/xarray/issues/3751>`_, :issue:`32905`).
 - Bug in :class:`Index` constructor where an unhelpful error message was raised for ``numpy`` scalars (:issue:`33017`)
 - Bug in :meth:`DataFrame.lookup` incorrectly raising an ``AttributeError`` when ``frame.index`` or ``frame.columns`` is not unique; this will now raise a ``ValueError`` with a helpful error message (:issue:`33041`)
-- Fixed regression in :meth:`DataFrame.loc`, :meth:`Series.loc` throwing an error when a ``datetime64[ns, tz]`` value is provided (:issue:`32395`)
+- Bug in :meth:`DataFrame.iloc.__setitem__` creating a new array instead of overwriting ``Categorical`` values in-place (:issue:`32831`)
 
 Missing
 ^^^^^^^

pandas/core/internals/blocks.py

@@ -818,8 +818,6 @@ def setitem(self, indexer, value):
 
         # coerce if block dtype can store value
         values = self.values
-        if is_object_dtype(values.dtype) and isinstance(value, DatetimeArray):
-            value = value.astype(object)
 
         if self._can_hold_element(value):
             # We only get here for non-Extension Blocks, so _try_coerce_args
@@ -852,8 +850,10 @@ def setitem(self, indexer, value):
         if is_extension_array_dtype(getattr(value, "dtype", None)):
             # We need to be careful not to allow through strings that
             #  can be parsed to EADtypes
+            is_ea_value = True
             arr_value = value
         else:
+            is_ea_value = False
             arr_value = np.array(value)
 
         if transpose:
@@ -881,6 +881,11 @@ def setitem(self, indexer, value):
             values[indexer] = value
             return self.make_block(Categorical(self.values, dtype=arr_value.dtype))
 
+        elif exact_match and is_ea_value:
+            # GH#?32395 if we're going to replace the values entirely, just
+            #  substitute in the new array
+            return self.make_block(arr_value)
+
         # if we are an exact match (ex-broadcasting),
         # then use the resultant dtype
         elif exact_match:

pandas/tests/computation/test_eval.py

@@ -314,7 +314,10 @@ def check_alignment(self, result, nlhs, ghs, op):
 
             # direct numpy comparison
             expected = self.ne.evaluate(f"nlhs {op} ghs")
-            tm.assert_numpy_array_equal(result.values, expected)
+            if np.issubdtype(result.values.dtype, np.floating):
+                tm.assert_almost_equal(result.values, expected)
+            else:
+                tm.assert_numpy_array_equal(result.values, expected)
 
     # modulus, pow, and floor division require special casing
 

pandas/tests/indexing/test_loc.py

@@ -1098,14 +1098,45 @@ def test_loc_datetimelike_mismatched_dtypes():
         df["a"].loc[tdi]
 
 
-def test_loc_setitem_df_datetime64tz_column_with_index():
+def test_loc_with_period_index_indexer():
+    # GH#4125
+    idx = pd.period_range("2002-01", "2003-12", freq="M")
+    df = pd.DataFrame(np.random.randn(24, 10), index=idx)
+    tm.assert_frame_equal(df, df.loc[idx])
+    tm.assert_frame_equal(df, df.loc[list(idx)])
+    tm.assert_frame_equal(df, df.loc[list(idx)])
+    tm.assert_frame_equal(df.iloc[0:5], df.loc[idx[0:5]])
+    tm.assert_frame_equal(df, df.loc[list(idx)])
+
+
+def test_loc_setitem_df_datetime64tz_full_column_with_index():
     df = pd.DataFrame(
         pd.date_range("2020-01-01", "2020-01-06", 6, tz=timezone.utc), columns=["data"]
     )
     df2 = pd.DataFrame(index=df.index)
     df2.loc[df.index, "data"] = df["data"]
-    tm.assert_numpy_array_equal(np.array(df.data), np.array(df2.data))
-    assert df2.data.dtype == np.object
+    tm.assert_frame_equal(df, df2)
+
+
+def test_loc_setitem_df_datetime64tz_slice():
+    df = pd.DataFrame(
+        pd.date_range("2020-01-01", "2020-01-06", 6, tz=timezone.utc), columns=["data"]
+    )
+    df2 = pd.DataFrame(index=df.index)
+    expected = pd.DataFrame(
+        [
+            pd.Timestamp("2020-01-01", tz=timezone.utc),
+            pd.Timestamp("2020-01-02", tz=timezone.utc),
+            pd.Timestamp("2020-01-03", tz=timezone.utc),
+            pd.NaT,
+            pd.NaT,
+            pd.NaT,
+        ],
+        columns=["data"],
+        dtype="object",
+    )
+    df2.loc[df.index[:3], "data"] = df["data"][:3]
+    tm.assert_frame_equal(df2, expected)
 
 
 def test_loc_setitem_df_datetime64tz_column_without_index():
@@ -1114,24 +1145,110 @@ def test_loc_setitem_df_datetime64tz_column_without_index():
     )
     df2 = pd.DataFrame(index=df.index)
     df2.loc[:, "data"] = df["data"]
-    tm.assert_series_equal(df.data, df2.data)
+    tm.assert_frame_equal(df, df2)
 
 
-def test_loc_setitem_series_datetime64tz_with_index():
+def test_loc_setitem_series_datetime64tz_full_with_index():
     s1 = pd.Series(
         pd.date_range("2020-01-01", "2020-01-06", 6, tz=timezone.utc), name="data"
     )
-    s2 = pd.Series(index=s1.index, dtype=np.object, name="data")
+    s2 = pd.Series(index=s1.index, dtype="object", name="data")
     s2.loc[s1.index] = s1
-    tm.assert_numpy_array_equal(np.array(s1), np.array(s2))
-    assert s2.dtype == np.object
+    tm.assert_series_equal(s1, s2)
+
+
+def test_loc_setitem_series_datetime64tz_slice():
+    s1 = pd.Series(
+        pd.date_range("2020-01-01", "2020-01-06", 6, tz=timezone.utc), name="data"
+    )
+    s2 = pd.Series(index=s1.index, dtype="object", name="data")
+    expected = pd.Series(
+        [
+            *pd.date_range("2020-01-01", "2020-01-03", 3, tz=timezone.utc),
+            np.nan,
+            np.nan,
+            np.nan,
+        ],
+        name="data",
+        dtype="object",
+    )
+    s2.loc[s1.index[:3]] = s1[:3]
+    tm.assert_series_equal(expected, s2)
 
 
 def test_loc_setitem_series_datetime64tz_without_index():
     s1 = pd.Series(
         pd.date_range("2020-01-01", "2020-01-06", 6, tz=timezone.utc), name="data"
     )
-    s2 = pd.Series(index=s1.index, dtype=np.object, name="data")
+    s2 = pd.Series(index=s1.index, dtype="object", name="data")
     s2.loc[:] = s1
-    tm.assert_numpy_array_equal(np.array(s1), np.array(s2))
-    assert s2.dtype == np.object
+    tm.assert_series_equal(s1, s2)
+
+
+def test_loc_setitem_series_timedelta64_full_with_index():
+    s1 = pd.Series(pd.timedelta_range(start="1 day", periods=4), name="data")
+    s2 = pd.Series(index=s1.index, dtype="object", name="data")
+    s2.loc[s1.index] = s1
+    tm.assert_series_equal(s1, s2)
+
+
+def test_loc_setitem_df_period_full_column_with_index():
+    df = pd.DataFrame(pd.period_range(start="2020Q1", periods=5), columns=["data"])
+    df2 = pd.DataFrame(index=df.index)
+    df2.loc[df.index, "data"] = df["data"]
+    tm.assert_frame_equal(df, df2)
+
+
+def test_loc_setitem_series_interval_full_with_index():
+    s1 = pd.Series(pd.interval_range(start=0, end=5), name="data")
+    s2 = pd.Series(index=s1.index, dtype="object", name="data")
+    s2.loc[s1.index] = s1
+    tm.assert_series_equal(s1, s2)
+
+
+def test_loc_setitem_df_sparse_full_column_with_index():
+    df = pd.DataFrame(np.random.randn(100), columns=["data"])
+    df.iloc[5:95] = np.nan
+    df = df.astype(pd.SparseDtype("int", np.nan))
+    df2 = pd.DataFrame(index=df.index)
+    df2.loc[df.index, "data"] = df["data"]
+    tm.assert_frame_equal(df, df2)
+
+
+def test_loc_setitem_series_categorical_full_with_index():
+    s1 = pd.Series(pd.Categorical([1] * 10 + [2] * 5 + [3] * 10), name="data")
+    s2 = pd.Series(index=s1.index, dtype="object", name="data")
+    s2.loc[s1.index] = s1
+    tm.assert_series_equal(s1, s2)
+
+
+def test_loc_setitem_series_categorical_slice():
+    s1 = pd.Series(pd.Categorical([1] * 10 + [2] * 5 + [3] * 10), name="data")
+    s2 = pd.Series(index=s1.index, dtype="object", name="data")
+    expected = pd.Series(
+        pd.Categorical([1] * 10 + [2] * 5 + [np.nan] * 10), name="data", dtype=object
+    )
+    s2.loc[s1.index[:15]] = s1[:15]
+    tm.assert_series_equal(expected, s2)
+
+
+def test_loc_setitem_df_interval_slice():
+    df = pd.DataFrame(pd.interval_range(start=0, end=5), columns=["data"])
+    df2 = pd.DataFrame(index=df.index)
+    expected = pd.DataFrame(
+        [*pd.interval_range(start=0, end=3), np.nan, np.nan], columns=["data"]
+    )
+    df2.loc[df.index[:3], "data"] = df["data"][:3]
+    tm.assert_frame_equal(df2, expected)
+
+
+def test_loc_setitem_series_period_slice():
+    s1 = pd.Series(pd.period_range(start="2020Q1", periods=5), name="data")
+    s2 = pd.Series(index=s1.index, dtype="object", name="data")
+    expected = pd.Series(
+        [*pd.period_range(start="2020Q1", periods=3), np.nan, np.nan],
+        name="data",
+        dtype=object,
+    )
+    s2.loc[s1.index[:3]] = s1[:3]
+    tm.assert_series_equal(expected, s2)