Merge branch 'master' into qcut-null-int

Author: Daniel Saxton

doc/source/whatsnew/v1.0.1.rst

@@ -15,7 +15,7 @@ including other versions of pandas.
 
 Bug fixes
 ~~~~~~~~~
-
+- Bug in :meth:`GroupBy.apply` was raising ``TypeError`` if called with function which returned a non-pandas non-scalar object (e.g. a list) (:issue:`31441`)
 
 Categorical
 ^^^^^^^^^^^
@@ -25,8 +25,8 @@ Categorical
 
 Datetimelike
 ^^^^^^^^^^^^
--
--
+- Fixed regression in :meth:`to_datetime` when parsing non-nanosecond resolution datetimes (:issue:`31491`)
+- Fixed bug in :meth:`to_datetime` raising when ``cache=True`` and out-of-bound values are present (:issue:`31491`)
 
 Timedelta
 ^^^^^^^^^
@@ -68,8 +68,11 @@ Interval
 Indexing
 ^^^^^^^^
 
+- Fixed regression when indexing a ``Series`` or ``DataFrame`` indexed by ``DatetimeIndex`` with a slice containg a :class:`datetime.date` (:issue:`31501`)
+- Fixed regression in :class:`DataFrame` setting values with a slice (e.g. ``df[-4:] = 1``) indexing by label instead of position (:issue:`31469`)
 -
 -
+- Bug where assigning to a :class:`Series` using a IntegerArray / BooleanArray as a mask would raise ``TypeError`` (:issue:`31446`)
 
 Missing
 ^^^^^^^

pandas/_libs/groupby.pyx

@@ -1143,7 +1143,7 @@ def group_rank(float64_t[:, :] out,
             # Update out only when there is a transition of values or labels.
             # When a new value or group is encountered, go back #dups steps(
             # the number of occurrence of current value) and assign the ranks
-            # based on the the starting index of the current group (grp_start)
+            # based on the starting index of the current group (grp_start)
             # and the current index
             if (i == N - 1 or
                     (masked_vals[_as[i]] != masked_vals[_as[i+1]]) or

pandas/_libs/reduction.pyx

@@ -501,9 +501,9 @@ def apply_frame_axis0(object frame, object f, object names,
 
             if not is_scalar(piece):
                 # Need to copy data to avoid appending references
-                if hasattr(piece, "copy"):
+                try:
                     piece = piece.copy(deep="all")
-                else:
+                except (TypeError, AttributeError):
                     piece = copy(piece)
 
             results.append(piece)

pandas/conftest.py

@@ -868,6 +868,16 @@ def tick_classes(request):
     )
 
 
+@pytest.fixture
+def datetime_series():
+    """
+    Fixture for Series of floats with DatetimeIndex
+    """
+    s = tm.makeTimeSeries()
+    s.name = "ts"
+    return s
+
+
 @pytest.fixture
 def float_frame():
     """

pandas/core/algorithms.py

@@ -29,6 +29,7 @@
     is_categorical_dtype,
     is_complex_dtype,
     is_datetime64_any_dtype,
+    is_datetime64_dtype,
     is_datetime64_ns_dtype,
     is_extension_array_dtype,
     is_float_dtype,
@@ -191,6 +192,11 @@ def _reconstruct_data(values, dtype, original):
         if isinstance(original, ABCIndexClass):
             values = values.astype(object, copy=False)
     elif dtype is not None:
+        if is_datetime64_dtype(dtype):
+            dtype = "datetime64[ns]"
+        elif is_timedelta64_dtype(dtype):
+            dtype = "timedelta64[ns]"
+
         values = values.astype(dtype, copy=False)
 
     return values

pandas/core/arrays/boolean.py

@@ -26,6 +26,7 @@
 from pandas.core.dtypes.missing import isna, notna
 
 from pandas.core import nanops, ops
+from pandas.core.indexers import check_array_indexer
 
 from .masked import BaseMaskedArray
 
@@ -382,6 +383,7 @@ def __setitem__(self, key, value):
             value = value[0]
             mask = mask[0]
 
+        key = check_array_indexer(self, key)
         self._data[key] = value
         self._mask[key] = mask
 

pandas/core/arrays/categorical.py

@@ -2073,6 +2073,8 @@ def __setitem__(self, key, value):
 
         lindexer = self.categories.get_indexer(rvalue)
         lindexer = self._maybe_coerce_indexer(lindexer)
+
+        key = check_array_indexer(self, key)
         self._codes[key] = lindexer
 
     def _reverse_indexer(self) -> Dict[Hashable, np.ndarray]:

pandas/core/arrays/datetimelike.py

@@ -601,6 +601,8 @@ def __setitem__(
                 f"or array of those. Got '{type(value).__name__}' instead."
             )
             raise TypeError(msg)
+
+        key = check_array_indexer(self, key)
         self._data[key] = value
         self._maybe_clear_freq()
 

pandas/core/arrays/integer.py

@@ -25,6 +25,7 @@
 from pandas.core.dtypes.missing import isna
 
 from pandas.core import nanops, ops
+from pandas.core.indexers import check_array_indexer
 from pandas.core.ops import invalid_comparison
 from pandas.core.ops.common import unpack_zerodim_and_defer
 from pandas.core.tools.numeric import to_numeric
@@ -425,6 +426,7 @@ def __setitem__(self, key, value):
             value = value[0]
             mask = mask[0]
 
+        key = check_array_indexer(self, key)
         self._data[key] = value
         self._mask[key] = mask
 

pandas/core/arrays/interval.py

@@ -541,6 +541,7 @@ def __setitem__(self, key, value):
                 msg = f"'value' should be an interval type, got {type(value)} instead."
                 raise TypeError(msg)
 
+        key = check_array_indexer(self, key)
         # Need to ensure that left and right are updated atomically, so we're
         # forced to copy, update the copy, and swap in the new values.
         left = self.left.copy(deep=True)
@@ -729,11 +730,11 @@ def _shallow_copy(self, left=None, right=None, closed=None):
         Parameters
         ----------
         left : array-like
-            Values to be used for the left-side of the the intervals.
+            Values to be used for the left-side of the intervals.
             If None, the existing left and right values will be used.
 
         right : array-like
-            Values to be used for the right-side of the the intervals.
+            Values to be used for the right-side of the intervals.
             If None and left is IntervalArray-like, the left and right
             of the IntervalArray-like will be used.
 

pandas/core/arrays/numpy_.py

@@ -262,6 +262,7 @@ def __getitem__(self, item):
     def __setitem__(self, key, value) -> None:
         value = extract_array(value, extract_numpy=True)
 
+        key = check_array_indexer(self, key)
         scalar_key = lib.is_scalar(key)
         scalar_value = lib.is_scalar(value)
 

pandas/core/arrays/period.py

@@ -33,6 +33,7 @@
 from pandas.core.dtypes.dtypes import PeriodDtype
 from pandas.core.dtypes.generic import (
     ABCIndexClass,
+    ABCPeriod,
     ABCPeriodArray,
     ABCPeriodIndex,
     ABCSeries,
@@ -960,8 +961,8 @@ def _get_ordinal_range(start, end, periods, freq, mult=1):
     if end is not None:
         end = Period(end, freq)
 
-    is_start_per = isinstance(start, Period)
-    is_end_per = isinstance(end, Period)
+    is_start_per = isinstance(start, ABCPeriod)
+    is_end_per = isinstance(end, ABCPeriod)
 
     if is_start_per and is_end_per and start.freq != end.freq:
         raise ValueError("start and end must have same freq")

pandas/core/arrays/string_.py

@@ -15,6 +15,7 @@
 from pandas.core import ops
 from pandas.core.arrays import PandasArray
 from pandas.core.construction import extract_array
+from pandas.core.indexers import check_array_indexer
 from pandas.core.missing import isna
 
 if TYPE_CHECKING:
@@ -238,6 +239,7 @@ def __setitem__(self, key, value):
             # extract_array doesn't extract PandasArray subclasses
             value = value._ndarray
 
+        key = check_array_indexer(self, key)
         scalar_key = lib.is_scalar(key)
         scalar_value = lib.is_scalar(value)
         if scalar_key and not scalar_value:

pandas/core/frame.py

@@ -2937,8 +2937,11 @@ def __setitem__(self, key, value):
             self._set_item(key, value)
 
     def _setitem_slice(self, key, value):
+        # NB: we can't just use self.loc[key] = value because that
+        #  operates on labels and we need to operate positional for
+        #  backwards-compat, xref GH#31469
         self._check_setitem_copy()
-        self.loc[key] = value
+        self.loc._setitem_with_indexer(key, value)
 
     def _setitem_array(self, key, value):
         # also raises Exception if object array with NA values

pandas/core/generic.py

@@ -2681,7 +2681,7 @@ def to_clipboard(
         ... # 0,1,2,3
         ... # 1,4,5,6
 
-        We can omit the the index by passing the keyword `index` and setting
+        We can omit the index by passing the keyword `index` and setting
         it to false.
 
         >>> df.to_clipboard(sep=',', index=False)