API: consistency with .ix and .loc for getitem operations (GH8613)

raise TypeError rather than KeyError on invalid scalar/slice indexing with that index type

Author: jreback

doc/source/indexing.rst

@@ -85,7 +85,7 @@ of multi-axis indexing.
 
 - ``.iloc`` is primarily integer position based (from ``0`` to
   ``length-1`` of the axis), but may also be used with a boolean
-  array.  ``.iloc`` will raise ``IndexError`` if a requested 
+  array.  ``.iloc`` will raise ``IndexError`` if a requested
   indexer is out-of-bounds, except *slice* indexers which allow
   out-of-bounds indexing.  (this conforms with python/numpy *slice*
   semantics).  Allowed inputs are:
@@ -292,6 +292,35 @@ Selection By Label
    This is sometimes called ``chained assignment`` and should be avoided.
    See :ref:`Returning a View versus Copy <indexing.view_versus_copy>`
 
+.. warning::
+
+   ``.loc`` is strict when you present slicers that are not compatible (or convertible) with the index type. For example
+   using integers in a ``DatetimeIndex`` or float indexers in an ``Int64Index``. These will raise a ``TypeError``.
+
+  .. ipython:: python
+
+     dfl = DataFrame(np.random.randn(5,4), columns=list('ABCD'), index=date_range('20130101',periods=5))
+     dfl
+     sl = Series(range(5),[-2,-1,1,2,3])
+     sl
+
+  .. code-block:: python
+
+     In [4]: dfl.loc[2:3]
+     TypeError: cannot do slice indexing on <class 'pandas.tseries.index.DatetimeIndex'> with these indexers [2] of <type 'int'>
+
+  .. code-block:: python
+
+     In [8]: sl.loc[-1.0:2]
+     TypeError: cannot do slice indexing on <class 'pandas.core.index.Int64Index'> with these indexers [-1.0] of <type 'float'>
+
+
+  String likes in slicing *can* be convertible to the type of the index and lead to natural slicing.
+
+  .. ipython:: python
+
+     dfl.loc['20130102':'20130104']
+
 pandas provides a suite of methods in order to have **purely label based indexing**. This is a strict inclusion based protocol.
 **at least 1** of the labels for which you ask, must be in the index or a ``KeyError`` will be raised! When slicing, the start bound is *included*, **AND** the stop bound is *included*. Integers are valid labels, but they refer to the label **and not the position**.
 
@@ -1486,5 +1515,3 @@ This will **not** work at all, and so should be avoided
    The chained assignment warnings / exceptions are aiming to inform the user of a possibly invalid
    assignment. There may be false positives; situations where a chained assignment is inadvertantly
    reported.
-
-

doc/source/whatsnew/v0.16.0.txt

@@ -211,6 +211,66 @@ Backwards incompatible API changes
      p // 0
 
 
+Indexing Changes
+~~~~~~~~~~~~~~~~
+
+.. _whatsnew_0160.api_breaking.indexing:
+
+The behavior of a small sub-set of edge cases for using ``.loc`` have changed (:issue:`8613`). Furthermore we have improved the content of the error messages that are raised:
+
+- slicing with ``.loc`` where the start and/or stop bound is not found in the index is now allowed; this previously would raise a ``KeyError``. This makes the behavior the same as ``.ix`` in this case. This change is only for slicing, not when indexing with a single label.
+
+  .. ipython:: python
+
+     df = DataFrame(np.random.randn(5,4), columns=list('ABCD'), index=date_range('20130101',periods=5))
+     df
+     s = Series(range(5),[-2,-1,1,2,3])
+     s
+
+  Previous Behavior
+
+  .. code-block:: python
+
+     In [4]: df.loc['2013-01-02':'2013-01-10']
+     KeyError: 'stop bound [2013-01-10] is not in the [index]'
+
+     In [6]: s.loc[-10:3]
+     KeyError: 'start bound [-10] is not the [index]'
+
+     In [8]: s.loc[-1.0:2]
+     Out[2]:
+     -1    1
+      1    2
+      2    3
+     dtype: int64
+
+  New Behavior
+
+  .. ipython:: python
+
+     df.loc['2013-01-02':'2013-01-10']
+     s.loc[-10:3]
+
+  .. code-block:: python
+
+     In [8]: s.loc[-1.0:2]
+     TypeError: cannot do slice indexing on <class 'pandas.core.index.Int64Index'> with these indexers [-1.0] of <type 'float'>
+
+- provide a useful exception for indexing with an invalid type for that index when using ``.loc``. For example trying to use ``.loc`` on an index of type ``DatetimeIndex`` or ``PeriodIndex`` or ``TimedeltaIndex``, with an integer (or a float).
+
+  Previous Behavior
+
+  .. code-block:: python
+
+     In [4]: df.loc[2:3]
+     KeyError: 'start bound [2] is not the [index]'
+
+  New Behavior
+
+  .. code-block:: python
+
+     In [4]: df.loc[2:3]
+     TypeError: Cannot do slice indexing on <class 'pandas.tseries.index.DatetimeIndex'> with <type 'int'> keys
 
 Deprecations
 ~~~~~~~~~~~~

pandas/core/index.py

@@ -640,7 +640,7 @@ def _convert_scalar_indexer(self, key, typ=None):
         def to_int():
             ikey = int(key)
             if ikey != key:
-                return self._convert_indexer_error(key, 'label')
+                return self._invalid_indexer('label', key)
             return ikey
 
         if typ == 'iloc':
@@ -651,7 +651,7 @@ def to_int():
                 warnings.warn("scalar indexers for index type {0} should be integers and not floating point".format(
                     type(self).__name__),FutureWarning)
                 return key
-            return self._convert_indexer_error(key, 'label')
+            return self._invalid_indexer('label', key)
 
         if is_float(key):
             if not self.is_floating():
@@ -667,7 +667,7 @@ def _validate_slicer(self, key, f):
 
         for c in ['start','stop','step']:
             if not f(getattr(key,c)):
-                self._convert_indexer_error(key.start, 'slice {0} value'.format(c))
+                self._invalid_indexer('slice {0} value'.format(c), key.start)
 
     def _convert_slice_indexer_getitem(self, key, is_index_slice=False):
         """ called from the getitem slicers, determine how to treat the key
@@ -698,7 +698,7 @@ def f(c):
                                   "and not floating point",FutureWarning)
                     return int(v)
 
-                self._convert_indexer_error(v, 'slice {0} value'.format(c))
+                self._invalid_indexer('slice {0} value'.format(c), v)
 
             return slice(*[ f(c) for c in ['start','stop','step']])
 
@@ -787,11 +787,13 @@ def _convert_list_indexer_for_mixed(self, keyarr, typ=None):
 
         return None
 
-    def _convert_indexer_error(self, key, msg=None):
-        if msg is None:
-            msg = 'label'
-        raise TypeError("the {0} [{1}] is not a proper indexer for this index "
-                        "type ({2})".format(msg, key, self.__class__.__name__))
+    def _invalid_indexer(self, form, key):
+        """ consistent invalid indexer message """
+        raise TypeError("cannot do {form} indexing on {klass} with these "
+                        "indexers [{key}] of {typ}".format(form=form,
+                                                           klass=type(self),
+                                                           key=key,
+                                                           typ=type(key)))
 
     def get_duplicates(self):
         from collections import defaultdict
@@ -2119,11 +2121,27 @@ def _maybe_cast_slice_bound(self, label, side):
         label : object
         side : {'left', 'right'}
 
+        Returns
+        -------
+        label :  object
+
         Notes
         -----
         Value of `side` parameter should be validated in caller.
 
         """
+
+        # pass thru float indexers if we have a numeric type index
+        # which then can decide to process / or convert and warng
+        if is_float(label):
+            if not self.is_floating():
+                self._invalid_indexer('slice',label)
+
+        # we are not an integer based index, and we have an integer label
+        # treat as positional based slicing semantics
+        if not self.is_integer() and is_integer(label):
+            self._invalid_indexer('slice',label)
+
         return label
 
     def _searchsorted_monotonic(self, label, side='left'):
@@ -2158,10 +2176,12 @@ def get_slice_bound(self, label, side):
                 " must be either 'left' or 'right': %s" % (side,))
 
         original_label = label
+
         # For datetime indices label may be a string that has to be converted
         # to datetime boundary according to its resolution.
         label = self._maybe_cast_slice_bound(label, side)
 
+        # we need to look up the label
         try:
             slc = self.get_loc(label)
         except KeyError as err:
@@ -2654,6 +2674,31 @@ def astype(self, dtype):
                             self.__class__)
         return Index(self.values, name=self.name, dtype=dtype)
 
+    def _maybe_cast_slice_bound(self, label, side):
+        """
+        This function should be overloaded in subclasses that allow non-trivial
+        casting on label-slice bounds, e.g. datetime-like indices allowing
+        strings containing formatted datetimes.
+
+        Parameters
+        ----------
+        label : object
+        side : {'left', 'right'}
+
+        Returns
+        -------
+        label :  object
+
+        Notes
+        -----
+        Value of `side` parameter should be validated in caller.
+
+        """
+        if not (is_integer(label) or is_float(label)):
+            self._invalid_indexer('slice',label)
+
+        return label
+
     def _convert_scalar_indexer(self, key, typ=None):
         if typ == 'iloc':
             return super(Float64Index, self)._convert_scalar_indexer(key,

pandas/core/indexing.py

@@ -1243,25 +1243,7 @@ def _has_valid_type(self, key, axis):
         # boolean
 
         if isinstance(key, slice):
-
-            if ax.is_floating():
-
-                # allowing keys to be slicers with no fallback
-                pass
-
-            else:
-                if key.start is not None:
-                    if key.start not in ax:
-                        raise KeyError(
-                            "start bound [%s] is not the [%s]" %
-                            (key.start, self.obj._get_axis_name(axis))
-                        )
-                if key.stop is not None:
-                    if key.stop not in ax:
-                        raise KeyError(
-                            "stop bound [%s] is not in the [%s]" %
-                            (key.stop, self.obj._get_axis_name(axis))
-                        )
+            return True
 
         elif is_bool_indexer(key):
             return True

pandas/core/series.py

@@ -536,7 +536,7 @@ def __getitem__(self, key):
             else:
 
                 # we can try to coerce the indexer (or this will raise)
-                new_key = self.index._convert_scalar_indexer(key)
+                new_key = self.index._convert_scalar_indexer(key,typ='getitem')
                 if type(new_key) != type(key):
                     return self.__getitem__(new_key)
                 raise

pandas/tests/test_index.py

@@ -950,16 +950,30 @@ def test_slice_locs(self):
             self.assertEqual(idx.slice_locs(start=3), (3, n))
             self.assertEqual(idx.slice_locs(3, 8), (3, 6))
             self.assertEqual(idx.slice_locs(5, 10), (3, n))
-            self.assertEqual(idx.slice_locs(5.0, 10.0), (3, n))
-            self.assertEqual(idx.slice_locs(4.5, 10.5), (3, 8))
             self.assertEqual(idx.slice_locs(end=8), (0, 6))
             self.assertEqual(idx.slice_locs(end=9), (0, 7))
 
+            # reversed
             idx2 = idx[::-1]
             self.assertEqual(idx2.slice_locs(8, 2), (2, 6))
-            self.assertEqual(idx2.slice_locs(8.5, 1.5), (2, 6))
             self.assertEqual(idx2.slice_locs(7, 3), (2, 5))
-            self.assertEqual(idx2.slice_locs(10.5, -1), (0, n))
+
+        # float slicing
+        idx = Index(np.array([0, 1, 2, 5, 6, 7, 9, 10], dtype=float))
+        n = len(idx)
+        self.assertEqual(idx.slice_locs(5.0, 10.0), (3, n))
+        self.assertEqual(idx.slice_locs(4.5, 10.5), (3, 8))
+        idx2 = idx[::-1]
+        self.assertEqual(idx2.slice_locs(8.5, 1.5), (2, 6))
+        self.assertEqual(idx2.slice_locs(10.5, -1), (0, n))
+
+        # int slicing with floats
+        idx = Index(np.array([0, 1, 2, 5, 6, 7, 9, 10], dtype=int))
+        self.assertRaises(TypeError, lambda : idx.slice_locs(5.0, 10.0))
+        self.assertRaises(TypeError, lambda : idx.slice_locs(4.5, 10.5))
+        idx2 = idx[::-1]
+        self.assertRaises(TypeError, lambda : idx2.slice_locs(8.5, 1.5))
+        self.assertRaises(TypeError, lambda : idx2.slice_locs(10.5, -1))
 
     def test_slice_locs_dup(self):
         idx = Index(['a', 'a', 'b', 'c', 'd', 'd'])