Merge remote-tracking branch 'upstream/master' into map-na-action

Author: dsaxton

doc/source/development/internals.rst

@@ -89,16 +89,10 @@ pandas extends NumPy's type system with custom types, like ``Categorical`` or
 datetimes with a timezone, so we have multiple notions of "values". For 1-D
 containers (``Index`` classes and ``Series``) we have the following convention:
 
-* ``cls._ndarray_values`` is *always* a NumPy ``ndarray``. Ideally,
-  ``_ndarray_values`` is cheap to compute. For example, for a ``Categorical``,
-  this returns the codes, not the array of objects.
 * ``cls._values`` refers is the "best possible" array. This could be an
-  ``ndarray``, ``ExtensionArray``, or in ``Index`` subclass (note: we're in the
-  process of removing the index subclasses here so that it's always an
-  ``ndarray`` or ``ExtensionArray``).
+  ``ndarray`` or ``ExtensionArray``.
 
-So, for example, ``Series[category]._values`` is a ``Categorical``, while
-``Series[category]._ndarray_values`` is the underlying codes.
+So, for example, ``Series[category]._values`` is a ``Categorical``.
 
 .. _ref-subclassing-pandas:
 

doc/source/reference/extensions.rst

@@ -37,7 +37,6 @@ objects.
       api.extensions.ExtensionArray._from_factorized
       api.extensions.ExtensionArray._from_sequence
       api.extensions.ExtensionArray._from_sequence_of_strings
-      api.extensions.ExtensionArray._ndarray_values
       api.extensions.ExtensionArray._reduce
       api.extensions.ExtensionArray._values_for_argsort
       api.extensions.ExtensionArray._values_for_factorize

doc/source/whatsnew/v1.1.0.rst

@@ -303,6 +303,7 @@ Indexing
 - Bug in :meth:`Series.loc` and :meth:`DataFrame.loc` when indexing with an integer key on a object-dtype :class:`Index` that is not all-integers (:issue:`31905`)
 - Bug in :meth:`DataFrame.iloc.__setitem__` on a :class:`DataFrame` with duplicate columns incorrectly setting values for all matching columns (:issue:`15686`, :issue:`22036`)
 - Bug in :meth:`DataFrame.loc:` and :meth:`Series.loc` with a :class:`DatetimeIndex`, :class:`TimedeltaIndex`, or :class:`PeriodIndex` incorrectly allowing lookups of non-matching datetime-like dtypes (:issue:`32650`)
+- Bug in :meth:`Series.__getitem__` indexing with non-standard scalars, e.g. ``np.dtype`` (:issue:`32684`)
 
 Missing
 ^^^^^^^

pandas/_libs/groupby.pyx

@@ -869,7 +869,9 @@ def group_last(rank_t[:, :] out,
 
     assert min_count == -1, "'min_count' only used in add and prod"
 
-    if not len(values) == len(labels):
+    # TODO(cython 3.0):
+    # Instead of `labels.shape[0]` use `len(labels)`
+    if not len(values) == labels.shape[0]:
         raise AssertionError("len(index) != len(labels)")
 
     nobs = np.zeros((<object>out).shape, dtype=np.int64)
@@ -960,7 +962,9 @@ def group_nth(rank_t[:, :] out,
 
     assert min_count == -1, "'min_count' only used in add and prod"
 
-    if not len(values) == len(labels):
+    # TODO(cython 3.0):
+    # Instead of `labels.shape[0]` use `len(labels)`
+    if not len(values) == labels.shape[0]:
         raise AssertionError("len(index) != len(labels)")
 
     nobs = np.zeros((<object>out).shape, dtype=np.int64)
@@ -1254,7 +1258,9 @@ def group_max(groupby_t[:, :] out,
 
     assert min_count == -1, "'min_count' only used in add and prod"
 
-    if not len(values) == len(labels):
+    # TODO(cython 3.0):
+    # Instead of `labels.shape[0]` use `len(labels)`
+    if not len(values) == labels.shape[0]:
         raise AssertionError("len(index) != len(labels)")
 
     nobs = np.zeros((<object>out).shape, dtype=np.int64)
@@ -1327,7 +1333,9 @@ def group_min(groupby_t[:, :] out,
 
     assert min_count == -1, "'min_count' only used in add and prod"
 
-    if not len(values) == len(labels):
+    # TODO(cython 3.0):
+    # Instead of `labels.shape[0]` use `len(labels)`
+    if not len(values) == labels.shape[0]:
         raise AssertionError("len(index) != len(labels)")
 
     nobs = np.zeros((<object>out).shape, dtype=np.int64)

pandas/_libs/tslibs/strptime.pyx

@@ -4,7 +4,7 @@ import time
 import locale
 import calendar
 import re
-from datetime import date as datetime_date
+import datetime
 
 from _thread import allocate_lock as _thread_allocate_lock
 
@@ -288,20 +288,20 @@ def array_strptime(object[:] values, object fmt, bint exact=True, errors='raise'
             elif iso_year != -1 and iso_week != -1:
                 year, julian = _calc_julian_from_V(iso_year, iso_week,
                                                    weekday + 1)
-        # Cannot pre-calculate datetime_date() since can change in Julian
+        # Cannot pre-calculate datetime.date() since can change in Julian
         # calculation and thus could have different value for the day of the wk
         # calculation.
         try:
             if julian == -1:
                 # Need to add 1 to result since first day of the year is 1, not
                 # 0.
-                ordinal = datetime_date(year, month, day).toordinal()
-                julian = ordinal - datetime_date(year, 1, 1).toordinal() + 1
+                ordinal = datetime.date(year, month, day).toordinal()
+                julian = ordinal - datetime.date(year, 1, 1).toordinal() + 1
             else:
                 # Assume that if they bothered to include Julian day it will
                 # be accurate.
-                datetime_result = datetime_date.fromordinal(
-                    (julian - 1) + datetime_date(year, 1, 1).toordinal())
+                datetime_result = datetime.date.fromordinal(
+                    (julian - 1) + datetime.date(year, 1, 1).toordinal())
                 year = datetime_result.year
                 month = datetime_result.month
                 day = datetime_result.day
@@ -311,7 +311,7 @@ def array_strptime(object[:] values, object fmt, bint exact=True, errors='raise'
                 continue
             raise
         if weekday == -1:
-            weekday = datetime_date(year, month, day).weekday()
+            weekday = datetime.date(year, month, day).weekday()
 
         dts.year = year
         dts.month = month
@@ -649,7 +649,7 @@ cdef int _calc_julian_from_U_or_W(int year, int week_of_year,
     cdef:
         int first_weekday, week_0_length, days_to_week
 
-    first_weekday = datetime_date(year, 1, 1).weekday()
+    first_weekday = datetime.date(year, 1, 1).weekday()
     # If we are dealing with the %U directive (week starts on Sunday), it's
     # easier to just shift the view to Sunday being the first day of the
     # week.
@@ -692,14 +692,14 @@ cdef (int, int) _calc_julian_from_V(int iso_year, int iso_week, int iso_weekday)
     cdef:
         int correction, ordinal
 
-    correction = datetime_date(iso_year, 1, 4).isoweekday() + 3
+    correction = datetime.date(iso_year, 1, 4).isoweekday() + 3
     ordinal = (iso_week * 7) + iso_weekday - correction
     # ordinal may be negative or 0 now, which means the date is in the previous
     # calendar year
     if ordinal < 1:
-        ordinal += datetime_date(iso_year, 1, 1).toordinal()
+        ordinal += datetime.date(iso_year, 1, 1).toordinal()
         iso_year -= 1
-        ordinal -= datetime_date(iso_year, 1, 1).toordinal()
+        ordinal -= datetime.date(iso_year, 1, 1).toordinal()
     return iso_year, ordinal
 
 

pandas/core/array_algos/__init__.py

@@ -0,0 +1,9 @@
+"""
+core.array_algos is for algorithms that operate on ndarray and ExtensionArray.
+These should:
+
+- Assume that any Index, Series, or DataFrame objects have already been unwrapped.
+- Assume that any list arguments have already been cast to ndarray/EA.
+- Not depend on Index, Series, or DataFrame, nor import any of these.
+- May dispatch to ExtensionArray methods, but should not import from core.arrays.
+"""

pandas/core/array_algos/transforms.py

@@ -0,0 +1,33 @@
+"""
+transforms.py is for shape-preserving functions.
+"""
+
+import numpy as np
+
+from pandas.core.dtypes.common import ensure_platform_int
+
+
+def shift(values: np.ndarray, periods: int, axis: int, fill_value) -> np.ndarray:
+    new_values = values
+
+    # make sure array sent to np.roll is c_contiguous
+    f_ordered = values.flags.f_contiguous
+    if f_ordered:
+        new_values = new_values.T
+        axis = new_values.ndim - axis - 1
+
+    if np.prod(new_values.shape):
+        new_values = np.roll(new_values, ensure_platform_int(periods), axis=axis)
+
+    axis_indexer = [slice(None)] * values.ndim
+    if periods > 0:
+        axis_indexer[axis] = slice(None, periods)
+    else:
+        axis_indexer[axis] = slice(periods, None)
+    new_values[tuple(axis_indexer)] = fill_value
+
+    # restore original order
+    if f_ordered:
+        new_values = new_values.T
+
+    return new_values

pandas/core/arrays/base.py

@@ -93,7 +93,6 @@ class ExtensionArray:
     _from_factorized
     _from_sequence
     _from_sequence_of_strings
-    _ndarray_values
     _reduce
     _values_for_argsort
     _values_for_factorize
@@ -1046,22 +1045,6 @@ def _concat_same_type(
     # of objects
     _can_hold_na = True
 
-    @property
-    def _ndarray_values(self) -> np.ndarray:
-        """
-        Internal pandas method for lossy conversion to a NumPy ndarray.
-
-        This method is not part of the pandas interface.
-
-        The expectation is that this is cheap to compute, and is primarily
-        used for interacting with our indexers.
-
-        Returns
-        -------
-        array : ndarray
-        """
-        return np.array(self)
-
     def _reduce(self, name, skipna=True, **kwargs):
         """
         Return a scalar result of performing the reduction operation.

pandas/core/arrays/categorical.py

@@ -451,10 +451,6 @@ def dtype(self) -> CategoricalDtype:
         """
         return self._dtype
 
-    @property
-    def _ndarray_values(self) -> np.ndarray:
-        return self.codes
-
     @property
     def _constructor(self) -> Type["Categorical"]:
         return Categorical
@@ -2567,12 +2563,7 @@ def _get_codes_for_values(values, categories):
     """
     dtype_equal = is_dtype_equal(values.dtype, categories.dtype)
 
-    if dtype_equal:
-        # To prevent erroneous dtype coercion in _get_data_algo, retrieve
-        # the underlying numpy array. gh-22702
-        values = getattr(values, "_ndarray_values", values)
-        categories = getattr(categories, "_ndarray_values", categories)
-    elif is_extension_array_dtype(categories.dtype) and is_object_dtype(values):
+    if is_extension_array_dtype(categories.dtype) and is_object_dtype(values):
         # Support inferring the correct extension dtype from an array of
         # scalar objects. e.g.
         # Categorical(array[Period, Period], categories=PeriodIndex(...))
@@ -2582,7 +2573,7 @@ def _get_codes_for_values(values, categories):
             # exception raised in _from_sequence
             values = ensure_object(values)
             categories = ensure_object(categories)
-    else:
+    elif not dtype_equal:
         values = ensure_object(values)
         categories = ensure_object(categories)
 

pandas/core/arrays/datetimelike.py

@@ -40,6 +40,7 @@
 
 from pandas.core import missing, nanops, ops
 from pandas.core.algorithms import checked_add_with_arr, take, unique1d, value_counts
+from pandas.core.array_algos.transforms import shift
 from pandas.core.arrays.base import ExtensionArray, ExtensionOpsMixin
 import pandas.core.common as com
 from pandas.core.construction import array, extract_array
@@ -455,10 +456,6 @@ def asi8(self) -> np.ndarray:
         # do not cache or you'll create a memory leak
         return self._data.view("i8")
 
-    @property
-    def _ndarray_values(self):
-        return self._data
-
     # ----------------------------------------------------------------
     # Rendering Methods
 
@@ -773,26 +770,7 @@ def shift(self, periods=1, fill_value=None, axis=0):
 
         fill_value = self._unbox_scalar(fill_value)
 
-        new_values = self._data
-
-        # make sure array sent to np.roll is c_contiguous
-        f_ordered = new_values.flags.f_contiguous
-        if f_ordered:
-            new_values = new_values.T
-            axis = new_values.ndim - axis - 1
-
-        new_values = np.roll(new_values, periods, axis=axis)
-
-        axis_indexer = [slice(None)] * self.ndim
-        if periods > 0:
-            axis_indexer[axis] = slice(None, periods)
-        else:
-            axis_indexer[axis] = slice(periods, None)
-        new_values[tuple(axis_indexer)] = fill_value
-
-        # restore original order
-        if f_ordered:
-            new_values = new_values.T
+        new_values = shift(self._data, periods, axis, fill_value)
 
         return type(self)._simple_new(new_values, dtype=self.dtype)
 

pandas/core/arrays/integer.py

@@ -478,18 +478,6 @@ def astype(self, dtype, copy: bool = True) -> ArrayLike:
         data = self.to_numpy(dtype=dtype, **kwargs)
         return astype_nansafe(data, dtype, copy=False)
 
-    @property
-    def _ndarray_values(self) -> np.ndarray:
-        """
-        Internal pandas method for lossy conversion to a NumPy ndarray.
-
-        This method is not part of the pandas interface.
-
-        The expectation is that this is cheap to compute, and is primarily
-        used for interacting with our indexers.
-        """
-        return self._data
-
     def _values_for_factorize(self) -> Tuple[np.ndarray, float]:
         # TODO: https://github.com/pandas-dev/pandas/issues/30037
         # use masked algorithms, rather than object-dtype / np.nan.

pandas/core/base.py

@@ -855,23 +855,6 @@ def to_numpy(self, dtype=None, copy=False, na_value=lib.no_default, **kwargs):
                 result[self.isna()] = na_value
         return result
 
-    @property
-    def _ndarray_values(self) -> np.ndarray:
-        """
-        The data as an ndarray, possibly losing information.
-
-        The expectation is that this is cheap to compute, and is primarily
-        used for interacting with our indexers.
-
-        - categorical -> codes
-        """
-        if is_extension_array_dtype(self):
-            return self.array._ndarray_values
-        # As a mixin, we depend on the mixing class having values.
-        # Special mixin syntax may be developed in the future:
-        # https://github.com/python/typing/issues/246
-        return self.values  # type: ignore
-
     @property
     def empty(self):
         return not self.size