Merge pull request #9258 from shoyer/get_nearest

API/ENH: add method='nearest' to Index.get_indexer/reindex and method to get_loc

Author: shoyer

doc/source/basics.rst

@@ -948,15 +948,9 @@ chosen from the following table:
 
     pad / ffill, Fill values forward
     bfill / backfill, Fill values backward
+    nearest, Fill from the nearest index value
 
-Other fill methods could be added, of course, but these are the two most
-commonly used for time series data. In a way they only make sense for time
-series or otherwise ordered data, but you may have an application on non-time
-series data where this sort of "interpolation" logic is the correct thing to
-do. More sophisticated interpolation of missing values would be an obvious
-extension.
-
-We illustrate these fill methods on a simple TimeSeries:
+We illustrate these fill methods on a simple Series:
 
 .. ipython:: python
 
@@ -969,18 +963,22 @@ We illustrate these fill methods on a simple TimeSeries:
    ts2.reindex(ts.index)
    ts2.reindex(ts.index, method='ffill')
    ts2.reindex(ts.index, method='bfill')
+   ts2.reindex(ts.index, method='nearest')
 
-Note these methods require that the indexes are **order increasing**.
+These methods require that the indexes are **ordered** increasing or
+decreasing.
 
-Note the same result could have been achieved using :ref:`fillna
-<missing_data.fillna>`:
+Note that the same result could have been achieved using
+:ref:`fillna <missing_data.fillna>` (except for ``method='nearest'``) or
+:ref:`interpolate <missing_data.interpolation>`:
 
 .. ipython:: python
 
    ts2.reindex(ts.index).fillna(method='ffill')
 
-Note that ``reindex`` will raise a ValueError if the index is not
-monotonic. ``fillna`` will not make any checks on the order of the index.
+``reindex`` will raise a ValueError if the index is not monotonic increasing or
+descreasing. ``fillna`` and ``interpolate`` will not make any checks on the
+order of the index.
 
 .. _basics.drop:
 

doc/source/whatsnew/v0.16.0.txt

@@ -20,6 +20,15 @@ users upgrade to this version.
 New features
 ~~~~~~~~~~~~
 
+- Reindex now supports ``method='nearest'`` for frames or series with a monotonic increasing or decreasing index (:issue:`9258`):
+
+    .. ipython:: python
+
+    df = pd.DataFrame({'x': range(5)})
+    df.reindex([0.2, 1.8, 3.5], method='nearest')
+
+  This method is also exposed by the lower level ``Index.get_indexer`` and ``Index.get_loc`` methods.
+
 .. _whatsnew_0160.api:
 
 .. _whatsnew_0160.api_breaking:
@@ -189,6 +198,9 @@ Enhancements
 
 - Added ``StringMethods.find()`` and ``rfind()`` which behave as the same as standard ``str`` (:issue:`9386`)
 
+- ``Index.get_indexer`` now supports ``method='pad'`` and ``method='backfill'`` even for any target array, not just monotonic targets. These methods also work for monotonic decreasing as well as monotonic increasing indexes (:issue:`9258`).
+- ``Index.asof`` now works on all index types (:issue:`9258`).
+
 - Added ``StringMethods.isnumeric`` and ``isdecimal`` which behave as the same as standard ``str`` (:issue:`9439`)
 - The ``read_excel()`` function's :ref:`sheetname <_io.specifying_sheets>` argument now accepts a list and ``None``, to get multiple or all sheets respectively.  If more than one sheet is specified, a dictionary is returned. (:issue:`9450`)
 
@@ -252,6 +264,22 @@ Bug Fixes
 
 - Fixed character encoding bug in ``read_stata`` and ``StataReader`` when loading data from a URL (:issue:`9231`).
 
+- Looking up a partial string label with ``DatetimeIndex.asof`` now includes values that match the string, even if they are after the start of the partial string label (:issue:`9258`). Old behavior:
+
+  .. ipython:: python
+    :verbatim:
+
+    In [4]: pd.to_datetime(['2000-01-31', '2000-02-28']).asof('2000-02')
+    Out[4]: Timestamp('2000-01-31 00:00:00')
+
+  Fixed behavior:
+
+  .. ipython:: python
+
+    pd.to_datetime(['2000-01-31', '2000-02-28']).asof('2000-02')
+
+  To reproduce the old behavior, simply add more precision to the label (e.g., use ``2000-02-01`` instead of ``2000-02``).
+
 
 
 - Bug in adding ``offsets.Nano`` to other offets raises ``TypeError`` (:issue:`9284`)

pandas/core/common.py

@@ -2682,21 +2682,31 @@ def _astype_nansafe(arr, dtype, copy=True):
     return arr.view(dtype)
 
 
-def _clean_fill_method(method):
+def _clean_fill_method(method, allow_nearest=False):
     if method is None:
         return None
     method = method.lower()
     if method == 'ffill':
         method = 'pad'
     if method == 'bfill':
         method = 'backfill'
-    if method not in ['pad', 'backfill']:
-        msg = ('Invalid fill method. Expecting pad (ffill) or backfill '
-               '(bfill). Got %s' % method)
+
+    valid_methods = ['pad', 'backfill']
+    expecting = 'pad (ffill) or backfill (bfill)'
+    if allow_nearest:
+        valid_methods.append('nearest')
+        expecting = 'pad (ffill), backfill (bfill) or nearest'
+    if method not in valid_methods:
+        msg = ('Invalid fill method. Expecting %s. Got %s'
+               % (expecting, method))
         raise ValueError(msg)
     return method
 
 
+def _clean_reindex_fill_method(method):
+    return _clean_fill_method(method, allow_nearest=True)
+
+
 def _all_none(*args):
     for arg in args:
         if arg is not None:

pandas/core/generic.py

@@ -1672,10 +1672,12 @@ def sort_index(self, axis=0, ascending=True):
             keywords)
             New labels / index to conform to. Preferably an Index object to
             avoid duplicating data
-        method : {'backfill', 'bfill', 'pad', 'ffill', None}, default None
-            Method to use for filling holes in reindexed DataFrame
-            pad / ffill: propagate last valid observation forward to next valid
-            backfill / bfill: use NEXT valid observation to fill gap
+        method : {None, 'backfill'/'bfill', 'pad'/'ffill', 'nearest'}, optional
+            Method to use for filling holes in reindexed DataFrame:
+              * default: don't fill gaps
+              * pad / ffill: propagate last valid observation forward to next valid
+              * backfill / bfill: use next valid observation to fill gap
+              * nearest: use nearest valid observations to fill gap
         copy : boolean, default True
             Return a new object, even if the passed indexes are the same
         level : int or name
@@ -1703,7 +1705,7 @@ def reindex(self, *args, **kwargs):
 
         # construct the args
         axes, kwargs = self._construct_axes_from_arguments(args, kwargs)
-        method = com._clean_fill_method(kwargs.get('method'))
+        method = com._clean_reindex_fill_method(kwargs.get('method'))
         level = kwargs.get('level')
         copy = kwargs.get('copy', True)
         limit = kwargs.get('limit')
@@ -1744,9 +1746,8 @@ def _reindex_axes(self, axes, level, limit, method, fill_value, copy):
 
             axis = self._get_axis_number(a)
             obj = obj._reindex_with_indexers(
-                {axis: [new_index, indexer]}, method=method,
-                fill_value=fill_value, limit=limit, copy=copy,
-                allow_dups=False)
+                {axis: [new_index, indexer]},
+                fill_value=fill_value, copy=copy, allow_dups=False)
 
         return obj
 
@@ -1770,10 +1771,12 @@ def _reindex_multi(self, axes, copy, fill_value):
             New labels / index to conform to. Preferably an Index object to
             avoid duplicating data
         axis : %(axes_single_arg)s
-        method : {'backfill', 'bfill', 'pad', 'ffill', None}, default None
-            Method to use for filling holes in reindexed object.
-            pad / ffill: propagate last valid observation forward to next valid
-            backfill / bfill: use NEXT valid observation to fill gap
+        method : {None, 'backfill'/'bfill', 'pad'/'ffill', 'nearest'}, optional
+            Method to use for filling holes in reindexed DataFrame:
+              * default: don't fill gaps
+              * pad / ffill: propagate last valid observation forward to next valid
+              * backfill / bfill: use next valid observation to fill gap
+              * nearest: use nearest valid observations to fill gap
         copy : boolean, default True
             Return a new object, even if the passed indexes are the same
         level : int or name
@@ -1802,15 +1805,14 @@ def reindex_axis(self, labels, axis=0, method=None, level=None, copy=True,
 
         axis_name = self._get_axis_name(axis)
         axis_values = self._get_axis(axis_name)
-        method = com._clean_fill_method(method)
+        method = com._clean_reindex_fill_method(method)
         new_index, indexer = axis_values.reindex(labels, method, level,
                                                  limit=limit)
         return self._reindex_with_indexers(
-            {axis: [new_index, indexer]}, method=method, fill_value=fill_value,
-            limit=limit, copy=copy)
+            {axis: [new_index, indexer]}, fill_value=fill_value, copy=copy)
 
-    def _reindex_with_indexers(self, reindexers, method=None,
-                               fill_value=np.nan, limit=None, copy=False,
+    def _reindex_with_indexers(self, reindexers,
+                               fill_value=np.nan, copy=False,
                                allow_dups=False):
         """ allow_dups indicates an internal call here """