Merge branch 'master' of https://github.com/pandas-dev/pandas into no-values

Author: jbrockmendel

doc/source/whatsnew/v1.0.3.rst

@@ -15,6 +15,7 @@ including other versions of pandas.
 
 Fixed regressions
 ~~~~~~~~~~~~~~~~~
+- Fixed regression in ``resample.agg`` when the underlying data is non-writeable (:issue:`31710`)
 
 .. _whatsnew_103.bug_fixes:
 

doc/source/whatsnew/v1.1.0.rst

@@ -301,6 +301,7 @@ Indexing
 - Bug in :meth:`DataFrame.iat` incorrectly returning ``Timestamp`` instead of ``datetime`` in some object-dtype cases (:issue:`32809`)
 - Bug in :meth:`Series.loc` and :meth:`DataFrame.loc` when indexing with an integer key on a object-dtype :class:`Index` that is not all-integers (:issue:`31905`)
 - Bug in :meth:`DataFrame.iloc.__setitem__` on a :class:`DataFrame` with duplicate columns incorrectly setting values for all matching columns (:issue:`15686`, :issue:`22036`)
+- Bug in :meth:`DataFrame.loc:` and :meth:`Series.loc` with a :class:`DatetimeIndex`, :class:`TimedeltaIndex`, or :class:`PeriodIndex` incorrectly allowing lookups of non-matching datetime-like dtypes (:issue:`32650`)
 
 Missing
 ^^^^^^^

pandas/_libs/groupby.pyx

@@ -848,11 +848,13 @@ cdef inline bint _treat_as_na(rank_t val, bint is_datetimelike) nogil:
         return val != val
 
 
+# GH#31710 use memorviews once cython 0.30 is released so we can
+#  use `const rank_t[:, :] values`
 @cython.wraparound(False)
 @cython.boundscheck(False)
 def group_last(rank_t[:, :] out,
                int64_t[:] counts,
-               rank_t[:, :] values,
+               ndarray[rank_t, ndim=2] values,
                const int64_t[:] labels,
                Py_ssize_t min_count=-1):
     """
@@ -937,11 +939,13 @@ def group_last(rank_t[:, :] out,
         raise RuntimeError("empty group with uint64_t")
 
 
+# GH#31710 use memorviews once cython 0.30 is released so we can
+#  use `const rank_t[:, :] values`
 @cython.wraparound(False)
 @cython.boundscheck(False)
 def group_nth(rank_t[:, :] out,
               int64_t[:] counts,
-              rank_t[:, :] values,
+              ndarray[rank_t, ndim=2] values,
               const int64_t[:] labels, int64_t rank=1,
               Py_ssize_t min_count=-1):
     """
@@ -1235,7 +1239,7 @@ ctypedef fused groupby_t:
 @cython.boundscheck(False)
 def group_max(groupby_t[:, :] out,
               int64_t[:] counts,
-              groupby_t[:, :] values,
+              ndarray[groupby_t, ndim=2] values,
               const int64_t[:] labels,
               Py_ssize_t min_count=-1):
     """
@@ -1308,7 +1312,7 @@ def group_max(groupby_t[:, :] out,
 @cython.boundscheck(False)
 def group_min(groupby_t[:, :] out,
               int64_t[:] counts,
-              groupby_t[:, :] values,
+              ndarray[groupby_t, ndim=2] values,
               const int64_t[:] labels,
               Py_ssize_t min_count=-1):
     """

pandas/_libs/src/ujson/python/objToJSON.c

@@ -222,13 +222,19 @@ static PyObject *get_values(PyObject *obj) {
 
     PRINTMARK();
 
-    if (PyObject_HasAttrString(obj, "_internal_get_values")) {
+    if (PyObject_TypeCheck(obj, cls_index) || PyObject_TypeCheck(obj, cls_series)) {
+        // The special cases to worry about are dt64tz and category[dt64tz].
+        //  In both cases we want the UTC-localized datetime64 ndarray,
+        //  without going through and object array of Timestamps.
         PRINTMARK();
-        values = PyObject_CallMethod(obj, "_internal_get_values", NULL);
+        values = PyObject_GetAttrString(obj, "values");
 
         if (values == NULL) {
             // Clear so we can subsequently try another method
             PyErr_Clear();
+        } else if (PyObject_HasAttrString(values, "__array__")) {
+            // We may have gotten a Categorical or Sparse array so call np.array
+            values = PyObject_CallMethod(values, "__array__", NULL);
         } else if (!PyArray_CheckExact(values)) {
             // Didn't get a numpy array, so keep trying
             PRINTMARK();

pandas/core/generic.py

@@ -5382,26 +5382,6 @@ def _values(self) -> np.ndarray:
         """internal implementation"""
         return self.values
 
-    def _internal_get_values(self) -> np.ndarray:
-        """
-        Return an ndarray after converting sparse values to dense.
-
-        This is the same as ``.values`` for non-sparse data. For sparse
-        data contained in a `SparseArray`, the data are first
-        converted to a dense representation.
-
-        Returns
-        -------
-        numpy.ndarray
-            Numpy representation of DataFrame.
-
-        See Also
-        --------
-        values : Numpy representation of DataFrame.
-        SparseArray : Container for sparse data.
-        """
-        return self.values
-
     @property
     def dtypes(self):
         """

pandas/core/indexes/base.py

@@ -29,7 +29,6 @@
     ensure_platform_int,
     is_bool,
     is_bool_dtype,
-    is_categorical,
     is_categorical_dtype,
     is_datetime64_any_dtype,
     is_dtype_equal,
@@ -532,6 +531,9 @@ def _shallow_copy_with_infer(self, values, **kwargs):
                 return self._constructor(values, **attributes)
             except (TypeError, ValueError):
                 pass
+
+        # Remove tz so Index will try non-DatetimeIndex inference
+        attributes.pop("tz", None)
         return Index(values, **attributes)
 
     def _update_inplace(self, result, **kwargs):
@@ -3870,50 +3872,6 @@ def _values(self) -> Union[ExtensionArray, np.ndarray]:
         """
         return self._data
 
-    def _internal_get_values(self) -> np.ndarray:
-        """
-        Return `Index` data as an `numpy.ndarray`.
-
-        Returns
-        -------
-        numpy.ndarray
-            A one-dimensional numpy array of the `Index` values.
-
-        See Also
-        --------
-        Index.values : The attribute that _internal_get_values wraps.
-
-        Examples
-        --------
-        Getting the `Index` values of a `DataFrame`:
-
-        >>> df = pd.DataFrame([[1, 2, 3], [4, 5, 6], [7, 8, 9]],
-        ...                    index=['a', 'b', 'c'], columns=['A', 'B', 'C'])
-        >>> df
-           A  B  C
-        a  1  2  3
-        b  4  5  6
-        c  7  8  9
-        >>> df.index._internal_get_values()
-        array(['a', 'b', 'c'], dtype=object)
-
-        Standalone `Index` values:
-
-        >>> idx = pd.Index(['1', '2', '3'])
-        >>> idx._internal_get_values()
-        array(['1', '2', '3'], dtype=object)
-
-        `MultiIndex` arrays also have only one dimension:
-
-        >>> midx = pd.MultiIndex.from_arrays([[1, 2, 3], ['a', 'b', 'c']],
-        ...                                  names=('number', 'letter'))
-        >>> midx._internal_get_values()
-        array([(1, 'a'), (2, 'b'), (3, 'c')], dtype=object)
-        >>> midx._internal_get_values().ndim
-        1
-        """
-        return self.values
-
     def _get_engine_target(self) -> np.ndarray:
         """
         Get the ndarray that we can pass to the IndexEngine constructor.
@@ -4657,10 +4615,8 @@ def get_indexer_non_unique(self, target):
         if pself is not self or ptarget is not target:
             return pself.get_indexer_non_unique(ptarget)
 
-        if is_categorical(target):
+        if is_categorical_dtype(target.dtype):
             tgt_values = np.asarray(target)
-        elif self.is_all_dates and target.is_all_dates:  # GH 30399
-            tgt_values = target.asi8
         else:
             tgt_values = target._get_engine_target()
 

pandas/core/indexes/datetimelike.py

@@ -8,13 +8,14 @@
 
 from pandas._libs import NaT, iNaT, join as libjoin, lib
 from pandas._libs.tslibs import timezones
-from pandas._typing import Label
+from pandas._typing import DtypeObj, Label
 from pandas.compat.numpy import function as nv
 from pandas.errors import AbstractMethodError
 from pandas.util._decorators import Appender, cache_readonly, doc
 
 from pandas.core.dtypes.common import (
     ensure_int64,
+    ensure_platform_int,
     is_bool_dtype,
     is_categorical_dtype,
     is_dtype_equal,
@@ -32,7 +33,7 @@
 from pandas.core.arrays.datetimelike import DatetimeLikeArrayMixin
 from pandas.core.base import IndexOpsMixin
 import pandas.core.indexes.base as ibase
-from pandas.core.indexes.base import Index, _index_shared_docs
+from pandas.core.indexes.base import Index, _index_shared_docs, ensure_index
 from pandas.core.indexes.extension import (
     ExtensionIndex,
     inherit_names,
@@ -101,6 +102,12 @@ class DatetimeIndexOpsMixin(ExtensionIndex):
     def is_all_dates(self) -> bool:
         return True
 
+    def _is_comparable_dtype(self, dtype: DtypeObj) -> bool:
+        """
+        Can we compare values of the given dtype to our own?
+        """
+        raise AbstractMethodError(self)
+
     # ------------------------------------------------------------------------
     # Abstract data attributes
 
@@ -426,6 +433,21 @@ def _partial_date_slice(
             # try to find the dates
             return (lhs_mask & rhs_mask).nonzero()[0]
 
+    @Appender(Index.get_indexer_non_unique.__doc__)
+    def get_indexer_non_unique(self, target):
+        target = ensure_index(target)
+        pself, ptarget = self._maybe_promote(target)
+        if pself is not self or ptarget is not target:
+            return pself.get_indexer_non_unique(ptarget)
+
+        if not self._is_comparable_dtype(target.dtype):
+            no_matches = -1 * np.ones(self.shape, dtype=np.intp)
+            return no_matches, no_matches
+
+        tgt_values = target.asi8
+        indexer, missing = self._engine.get_indexer_non_unique(tgt_values)
+        return ensure_platform_int(indexer), missing
+
     # --------------------------------------------------------------------
 
     __add__ = make_wrapped_arith_op("__add__")

pandas/core/indexes/datetimes.py

@@ -7,10 +7,18 @@
 
 from pandas._libs import NaT, Period, Timestamp, index as libindex, lib, tslib as libts
 from pandas._libs.tslibs import fields, parsing, timezones
-from pandas._typing import Label
+from pandas._typing import DtypeObj, Label
 from pandas.util._decorators import cache_readonly
 
-from pandas.core.dtypes.common import _NS_DTYPE, is_float, is_integer, is_scalar
+from pandas.core.dtypes.common import (
+    _NS_DTYPE,
+    is_datetime64_any_dtype,
+    is_datetime64_dtype,
+    is_datetime64tz_dtype,
+    is_float,
+    is_integer,
+    is_scalar,
+)
 from pandas.core.dtypes.missing import is_valid_nat_for_dtype
 
 from pandas.core.arrays.datetimes import DatetimeArray, tz_to_dtype
@@ -298,6 +306,18 @@ def _convert_for_op(self, value):
             return Timestamp(value).asm8
         raise ValueError("Passed item and index have different timezone")
 
+    def _is_comparable_dtype(self, dtype: DtypeObj) -> bool:
+        """
+        Can we compare values of the given dtype to our own?
+        """
+        if not is_datetime64_any_dtype(dtype):
+            return False
+        if self.tz is not None:
+            # If we have tz, we can compare to tzaware
+            return is_datetime64tz_dtype(dtype)
+        # if we dont have tz, we can only compare to tznaive
+        return is_datetime64_dtype(dtype)
+
     # --------------------------------------------------------------------
     # Rendering Methods
 

pandas/core/indexes/multi.py

@@ -3243,9 +3243,13 @@ def union(self, other, sort=None):
 
         # TODO: Index.union returns other when `len(self)` is 0.
 
-        uniq_tuples = lib.fast_unique_multiple(
-            [self._values, other._ndarray_values], sort=sort
-        )
+        if not is_object_dtype(other.dtype):
+            raise NotImplementedError(
+                "Can only union MultiIndex with MultiIndex or Index of tuples, "
+                "try mi.to_flat_index().union(other) instead."
+            )
+
+        uniq_tuples = lib.fast_unique_multiple([self._values, other._values], sort=sort)
 
         return MultiIndex.from_arrays(
             zip(*uniq_tuples), sortorder=0, names=result_names
@@ -3279,8 +3283,18 @@ def intersection(self, other, sort=False):
         if self.equals(other):
             return self
 
+        if not is_object_dtype(other.dtype):
+            # The intersection is empty
+            # TODO: we have no tests that get here
+            return MultiIndex(
+                levels=self.levels,
+                codes=[[]] * self.nlevels,
+                names=result_names,
+                verify_integrity=False,
+            )
+
         lvals = self._values
-        rvals = other._ndarray_values
+        rvals = other._values
 
         uniq_tuples = None  # flag whether _inner_indexer was succesful
         if self.is_monotonic and other.is_monotonic:

pandas/core/indexes/period.py

@@ -9,7 +9,7 @@
 from pandas._libs.tslibs import frequencies as libfrequencies, resolution
 from pandas._libs.tslibs.parsing import parse_time_string
 from pandas._libs.tslibs.period import Period
-from pandas._typing import Label
+from pandas._typing import DtypeObj, Label
 from pandas.util._decorators import Appender, cache_readonly
 
 from pandas.core.dtypes.common import (
@@ -23,6 +23,7 @@
     is_scalar,
     pandas_dtype,
 )
+from pandas.core.dtypes.dtypes import PeriodDtype
 
 from pandas.core.arrays.period import (
     PeriodArray,
@@ -298,6 +299,14 @@ def _maybe_convert_timedelta(self, other):
         # raise when input doesn't have freq
         raise raise_on_incompatible(self, None)
 
+    def _is_comparable_dtype(self, dtype: DtypeObj) -> bool:
+        """
+        Can we compare values of the given dtype to our own?
+        """
+        if not isinstance(dtype, PeriodDtype):
+            return False
+        return dtype.freq == self.freq
+
     # ------------------------------------------------------------------------
     # Rendering Methods
 
@@ -454,12 +463,11 @@ def get_indexer(self, target, method=None, limit=None, tolerance=None):
     def get_indexer_non_unique(self, target):
         target = ensure_index(target)
 
-        if isinstance(target, PeriodIndex):
-            if target.freq != self.freq:
-                no_matches = -1 * np.ones(self.shape, dtype=np.intp)
-                return no_matches, no_matches
+        if not self._is_comparable_dtype(target.dtype):
+            no_matches = -1 * np.ones(self.shape, dtype=np.intp)
+            return no_matches, no_matches
 
-            target = target.asi8
+        target = target.asi8
 
         indexer, missing = self._int64index.get_indexer_non_unique(target)
         return ensure_platform_int(indexer), missing