TST: GH4604, reindexing with a method of 'ffill' gives incorrect results

BUG/CLN: (GH4604) Refactor Series.reindex to core/generic.py allow method= in reindexing
  on a Series to work

API/CLN: GH4604 Infer and downcast dtype if appropriate on ffill/bfill
  this is for consistency when doing: df.reindex().ffill() and df.reindex(method='ffill')

CLN: allow backfill/pad/interpolate to operate on integers (by float conversion)
     provide downcasting back to original dtype where needed core.internals.interpolate

ENH: provide core.index.identical method to compare values and attributes similar to .equals

API: changed back to pre-GH3482 where a reindex with no args will by default copy

Author: jreback

doc/source/release.rst

@@ -144,8 +144,6 @@ See :ref:`Internal Refactoring<whatsnew_0130.refactoring>`
   - support attribute access for setting
   - filter supports same api as original ``DataFrame`` filter
 
-- Reindex called with no arguments will now return a copy of the input object
-
 - Series now inherits from ``NDFrame`` rather than directly from ``ndarray``.
   There are several minor changes that affect the API.
 
@@ -185,6 +183,11 @@ See :ref:`Internal Refactoring<whatsnew_0130.refactoring>`
 
 - Indexing with dtype conversions fixed (:issue:`4463`, :issue:`4204`)
 
+- Refactor Series.reindex to core/generic.py (:issue:`4604`), allow ``method=`` in reindexing
+  on a Series to work
+
+- Infer and downcast dtype if appropriate on ``ffill/bfill`` (:issue:`4604`)
+
 **Experimental Features**
 
 **Bug Fixes**
@@ -210,7 +213,7 @@ See :ref:`Internal Refactoring<whatsnew_0130.refactoring>`
   - In ``to_json``, raise if a passed ``orient`` would cause loss of data because
     of a duplicate index (:issue:`4359`)
   - In ``to_json``, fix date handling so milliseconds are the default timestamp
-    as the docstring says (:issue:`4362`). 
+    as the docstring says (:issue:`4362`).
   - JSON NaT handling fixed, NaTs are now serialised to `null` (:issue:`4498`)
   - Fixed passing ``keep_default_na=False`` when ``na_values=None`` (:issue:`4318`)
   - Fixed bug with ``values`` raising an error on a DataFrame with duplicate columns and mixed

doc/source/v0.13.0.txt

@@ -237,6 +237,11 @@ and behaviors. Series formerly subclassed directly from ``ndarray``. (:issue:`40
 
 - Indexing with dtype conversions fixed (:issue:`4463`, :issue:`4204`)
 
+- Refactor Series.reindex to core/generic.py (:issue:`4604`), allow ``method=`` in reindexing
+  on a Series to work
+
+- Infer and downcast dtype if appropriate on ``ffill/bfill`` (:issue:`4604`)
+
 Bug Fixes
 ~~~~~~~~~
 

pandas/core/common.py

@@ -961,16 +961,43 @@ def _possibly_downcast_to_dtype(result, dtype):
     """ try to cast to the specified dtype (e.g. convert back to bool/int
         or could be an astype of float64->float32 """
 
-    if np.isscalar(result):
+    if np.isscalar(result) or not len(result):
         return result
 
+    if isinstance(dtype, compat.string_types):
+        if dtype == 'infer':
+            inferred_type = lib.infer_dtype(_ensure_object(result.ravel()))
+            if inferred_type == 'boolean':
+                dtype = 'bool'
+            elif inferred_type == 'integer':
+                dtype = 'int64'
+            elif inferred_type == 'datetime64':
+                dtype = 'datetime64[ns]'
+            elif inferred_type == 'timedelta64':
+                dtype = 'timedelta64[ns]'
+
+            # try to upcast here
+            elif inferred_type == 'floating':
+                dtype = 'int64'
+
+            else:
+                dtype = 'object'
+
+    if isinstance(dtype, compat.string_types):
+        dtype = np.dtype(dtype)
+
     try:
         if issubclass(dtype.type, np.floating):
             return result.astype(dtype)
         elif dtype == np.bool_ or issubclass(dtype.type, np.integer):
-            if issubclass(result.dtype.type, np.number) and notnull(result).all():
+            if issubclass(result.dtype.type, (np.object_,np.number)) and notnull(result).all():
                 new_result = result.astype(dtype)
                 if (new_result == result).all():
+
+                    # a comparable, e.g. a Decimal may slip in here
+                    if not isinstance(result.ravel()[0], (np.integer,np.floating,np.bool,int,float,bool)):
+                        return result
+
                     return new_result
     except:
         pass
@@ -1052,6 +1079,9 @@ def pad_1d(values, limit=None, mask=None):
         _method = getattr(algos, 'pad_inplace_%s' % dtype, None)
     elif is_datetime64_dtype(values):
         _method = _pad_1d_datetime
+    elif is_integer_dtype(values):
+        values = _ensure_float64(values)
+        _method = algos.pad_inplace_float64
     elif values.dtype == np.object_:
         _method = algos.pad_inplace_object
 
@@ -1062,7 +1092,7 @@ def pad_1d(values, limit=None, mask=None):
         mask = isnull(values)
     mask = mask.view(np.uint8)
     _method(values, mask, limit=limit)
-
+    return values
 
 def backfill_1d(values, limit=None, mask=None):
 
@@ -1072,6 +1102,9 @@ def backfill_1d(values, limit=None, mask=None):
         _method = getattr(algos, 'backfill_inplace_%s' % dtype, None)
     elif is_datetime64_dtype(values):
         _method = _backfill_1d_datetime
+    elif is_integer_dtype(values):
+        values = _ensure_float64(values)
+        _method = algos.backfill_inplace_float64
     elif values.dtype == np.object_:
         _method = algos.backfill_inplace_object
 
@@ -1083,7 +1116,7 @@ def backfill_1d(values, limit=None, mask=None):
     mask = mask.view(np.uint8)
 
     _method(values, mask, limit=limit)
-
+    return values
 
 def pad_2d(values, limit=None, mask=None):
 
@@ -1093,6 +1126,9 @@ def pad_2d(values, limit=None, mask=None):
         _method = getattr(algos, 'pad_2d_inplace_%s' % dtype, None)
     elif is_datetime64_dtype(values):
         _method = _pad_2d_datetime
+    elif is_integer_dtype(values):
+        values = _ensure_float64(values)
+        _method = algos.pad_2d_inplace_float64
     elif values.dtype == np.object_:
         _method = algos.pad_2d_inplace_object
 
@@ -1108,7 +1144,7 @@ def pad_2d(values, limit=None, mask=None):
     else:
         # for test coverage
         pass
-
+    return values
 
 def backfill_2d(values, limit=None, mask=None):
 
@@ -1118,6 +1154,9 @@ def backfill_2d(values, limit=None, mask=None):
         _method = getattr(algos, 'backfill_2d_inplace_%s' % dtype, None)
     elif is_datetime64_dtype(values):
         _method = _backfill_2d_datetime
+    elif is_integer_dtype(values):
+        values = _ensure_float64(values)
+        _method = algos.backfill_2d_inplace_float64
     elif values.dtype == np.object_:
         _method = algos.backfill_2d_inplace_object
 
@@ -1133,7 +1172,7 @@ def backfill_2d(values, limit=None, mask=None):
     else:
         # for test coverage
         pass
-
+    return values
 
 def interpolate_2d(values, method='pad', axis=0, limit=None, missing=None):
     """ perform an actual interpolation of values, values will be make 2-d if needed
@@ -1153,10 +1192,11 @@ def interpolate_2d(values, method='pad', axis=0, limit=None, missing=None):
     else:  # todo create faster fill func without masking
         mask = mask_missing(transf(values), missing)
 
+    method = _clean_fill_method(method)
     if method == 'pad':
-        pad_2d(transf(values), limit=limit, mask=mask)
+        values = transf(pad_2d(transf(values), limit=limit, mask=mask))
     else:
-        backfill_2d(transf(values), limit=limit, mask=mask)
+        values = transf(backfill_2d(transf(values), limit=limit, mask=mask))
 
     # reshape back
     if ndim == 1:

pandas/core/generic.py

@@ -1003,11 +1003,15 @@ def reindex(self, *args, **kwargs):
             except:
                 pass
 
-        # perform the reindex on the axes
-        if copy and not com._count_not_none(*axes.values()):
-            return self.copy()
+        # if all axes that are requested to reindex are equal, then only copy if indicated
+        # must have index names equal here as well as values
+        if all([ self._get_axis(axis).identical(ax) for axis, ax in axes.items() if ax is not None ]):
+            if copy:
+                return self.copy()
+            return self
 
-        return self._reindex_axes(axes, level, limit, method, fill_value, copy, takeable=takeable)
+        # perform the reindex on the axes
+        return self._reindex_axes(axes, level, limit, method, fill_value, copy, takeable=takeable)._propogate_attributes(self)
 
     def _reindex_axes(self, axes, level, limit, method, fill_value, copy, takeable=False):
         """ perform the reinxed for all the axes """
@@ -1025,7 +1029,8 @@ def _reindex_axes(self, axes, level, limit, method, fill_value, copy, takeable=F
             new_index, indexer = self._get_axis(a).reindex(
                 labels, level=level, limit=limit, takeable=takeable)
             obj = obj._reindex_with_indexers(
-                {axis: [labels, indexer]}, method, fill_value, copy)
+                {axis: [new_index, indexer]}, method=method, fill_value=fill_value,
+                limit=limit, copy=copy)
 
         return obj
 
@@ -1079,21 +1084,26 @@ def reindex_axis(self, labels, axis=0, method=None, level=None, copy=True,
         axis_values = self._get_axis(axis_name)
         new_index, indexer = axis_values.reindex(labels, method, level,
                                                  limit=limit, copy_if_needed=True)
-        return self._reindex_with_indexers({axis: [new_index, indexer]}, method, fill_value, copy)
+        return self._reindex_with_indexers({axis: [new_index, indexer]}, method=method, fill_value=fill_value,
+                                           limit=limit, copy=copy)._propogate_attributes(self)
 
-    def _reindex_with_indexers(self, reindexers, method=None, fill_value=np.nan, copy=False):
+    def _reindex_with_indexers(self, reindexers, method=None, fill_value=np.nan, limit=None, copy=False):
 
         # reindex doing multiple operations on different axes if indiciated
         new_data = self._data
         for axis in sorted(reindexers.keys()):
             index, indexer = reindexers[axis]
             baxis = self._get_block_manager_axis(axis)
 
+            if index is None:
+                continue
+            index = _ensure_index(index)
+
             # reindex the axis
             if method is not None:
                 new_data = new_data.reindex_axis(
                     index, method=method, axis=baxis,
-                    fill_value=fill_value, copy=copy)
+                    fill_value=fill_value, limit=limit, copy=copy)
 
             elif indexer is not None:
                 # TODO: speed up on homogeneous DataFrame objects
@@ -1435,14 +1445,20 @@ def fillna(self, value=None, method=None, axis=0, inplace=False,
             if self._is_mixed_type and axis == 1:
                 if inplace:
                     raise NotImplementedError()
-                return self.T.fillna(method=method, limit=limit).T
+                result = self.T.fillna(method=method, limit=limit).T
+
+                # need to downcast here because of all of the transposes
+                result._data = result._data.downcast()
+
+                return result
 
             method = com._clean_fill_method(method)
             new_data = self._data.interpolate(method=method,
                                               axis=axis,
                                               limit=limit,
                                               inplace=inplace,
-                                              coerce=True)
+                                              coerce=True,
+                                              downcast=downcast)
         else:
             if method is not None:
                 raise ValueError('cannot specify both a fill method and value')
@@ -1474,11 +1490,11 @@ def fillna(self, value=None, method=None, axis=0, inplace=False,
 
     def ffill(self, axis=0, inplace=False, limit=None):
         return self.fillna(method='ffill', axis=axis, inplace=inplace,
-                           limit=limit)
+                           limit=limit, downcast='infer')
 
     def bfill(self, axis=0, inplace=False, limit=None):
         return self.fillna(method='bfill', axis=axis, inplace=inplace,
-                           limit=limit)
+                           limit=limit, downcast='infer')
 
     def replace(self, to_replace=None, value=None, inplace=False, limit=None,
                 regex=False, method=None, axis=None):

pandas/core/index.py

@@ -83,6 +83,7 @@ class Index(FrozenNDArray):
 
     name = None
     asi8 = None
+    _comparables = ['name']
 
     _engine_type = _index.ObjectEngine
 
@@ -545,6 +546,13 @@ def equals(self, other):
 
         return np.array_equal(self, other)
 
+    def identical(self, other):
+        """
+        Similar to equals, but check that other comparable attributes are also equal
+        """
+        return self.equals(other) and all(
+            [ getattr(self,c,None) == getattr(other,c,None) for c in self._comparables ])
+
     def asof(self, label):
         """
         For a sorted index, return the most recent label up to and including