ENH: 'replaced' series.replace with generic.replace !

CLN: cleaned up internal block action routines, now always return a list of blocks

Author: jreback

pandas/core/generic.py

@@ -1620,8 +1620,11 @@ def is_dictlike(x):
             return self.replace(to_replace, value, inplace=inplace,
                                 limit=limit, regex=regex)
         else:
-            if not len(self.columns):
-                return self
+
+            # need a non-zero len on all axes
+            for a in self._AXIS_ORDERS:
+                if not len(self._get_axis(a)):
+                    return self
 
             new_data = self._data
             if is_dictlike(to_replace):

pandas/core/internals.py

@@ -175,16 +175,6 @@ def itemsize(self):
     def dtype(self):
         return self.values.dtype
 
-    def copy(self, deep=True, ref_items=None):
-        values = self.values
-        if deep:
-            values = values.copy()
-        if ref_items is None:
-            ref_items = self.ref_items
-        return make_block(
-            values, self.items, ref_items, ndim=self.ndim, klass=self.__class__,
-            fastpath=True, placement=self._ref_locs)
-
     @property
     def ftype(self):
         return "%s:%s" % (self.dtype, self._ftype)
@@ -293,17 +283,23 @@ def split_block_at(self, item):
     def fillna(self, value, inplace=False, downcast=None):
         if not self._can_hold_na:
             if inplace:
-                return self
+                return [self]
             else:
-                return self.copy()
+                return [self.copy()]
 
         mask = com.isnull(self.values)
         value = self._try_fill(value)
         blocks = self.putmask(mask, value, inplace=inplace)
 
-        if downcast:
-            blocks = [ b.downcast() for b in blocks ]
-        return blocks
+        # possibily downcast the blocks
+        if not downcast:
+            return blocks
+
+        result_blocks = []
+        for b in blocks:
+            result_blocks.extend(b.downcast())
+
+        return result_blocks
 
     def downcast(self, dtypes=None):
         """ try to downcast each item to the dict of dtypes if present """
@@ -361,14 +357,14 @@ def _astype(self, dtype, copy=False, raise_on_error=True, values=None,
                                 "(%s [%s]) with smaller itemsize that current "
                                 "(%s [%s])" % (copy, self.dtype.name,
                                                self.itemsize, newb.dtype.name, newb.itemsize))
-        return newb
+        return [ newb ]
 
     def convert(self, copy=True, **kwargs):
         """ attempt to coerce any object types to better types
             return a copy of the block (if copy = True)
             by definition we are not an ObjectBlock here!  """
 
-        return self.copy() if copy else self
+        return [ self.copy() ] if copy else [ self ]
 
     def prepare_for_merge(self, **kwargs):
         """ a regular block is ok to merge as is """
@@ -428,6 +424,17 @@ def to_native_types(self, slicer=None, na_rep='', **kwargs):
         values[mask] = na_rep
         return values.tolist()
 
+    #### block actions ####
+    def copy(self, deep=True, ref_items=None):
+        values = self.values
+        if deep:
+            values = values.copy()
+        if ref_items is None:
+            ref_items = self.ref_items
+        return make_block(
+            values, self.items, ref_items, ndim=self.ndim, klass=self.__class__,
+            fastpath=True, placement=self._ref_locs)
+
     def replace(self, to_replace, value, inplace=False, filter=None,
                 regex=False):
         """ replace the to_replace value with value, possible to create new
@@ -541,7 +548,7 @@ def create_block(v, m, n, item, reshape=True):
         if inplace:
             return [self]
 
-        return make_block(new_values, self.items, self.ref_items, fastpath=True)
+        return [make_block(new_values, self.items, self.ref_items, fastpath=True)]
 
     def interpolate(self, method='pad', axis=0, inplace=False,
                     limit=None, missing=None, coerce=False):
@@ -551,20 +558,20 @@ def interpolate(self, method='pad', axis=0, inplace=False,
         if coerce:
             if not self._can_hold_na:
                 if inplace:
-                    return self
+                    return [self]
                 else:
-                    return self.copy()
+                    return [self.copy()]
 
         values = self.values if inplace else self.values.copy()
         values = com.interpolate_2d(values, method, axis, limit, missing)
-        return make_block(values, self.items, self.ref_items, ndim=self.ndim, klass=self.__class__, fastpath=True)
+        return [make_block(values, self.items, self.ref_items, ndim=self.ndim, klass=self.__class__, fastpath=True)]
 
     def take(self, indexer, ref_items, axis=1):
         if axis < 1:
             raise AssertionError('axis must be at least 1, got %d' % axis)
         new_values = com.take_nd(self.values, indexer, axis=axis,
                                  allow_fill=False)
-        return make_block(new_values, self.items, ref_items, ndim=self.ndim, klass=self.__class__, fastpath=True)
+        return [make_block(new_values, self.items, ref_items, ndim=self.ndim, klass=self.__class__, fastpath=True)]
 
     def get_values(self, dtype=None):
         return self.values
@@ -575,7 +582,7 @@ def get_merge_length(self):
     def diff(self, n):
         """ return block for the diff of the values """
         new_values = com.diff(self.values, n, axis=1)
-        return make_block(new_values, self.items, self.ref_items, ndim=self.ndim, fastpath=True)
+        return [make_block(new_values, self.items, self.ref_items, ndim=self.ndim, fastpath=True)]
 
     def shift(self, indexer, periods):
         """ shift the block by periods, possibly upcast """
@@ -588,7 +595,7 @@ def shift(self, indexer, periods):
             new_values[:, :periods] = fill_value
         else:
             new_values[:, periods:] = fill_value
-        return make_block(new_values, self.items, self.ref_items, ndim=self.ndim, fastpath=True)
+        return [make_block(new_values, self.items, self.ref_items, ndim=self.ndim, fastpath=True)]
 
     def eval(self, func, other, raise_on_error=True, try_cast=False):
         """
@@ -644,7 +651,7 @@ def eval(self, func, other, raise_on_error=True, try_cast=False):
         if try_cast:
             result = self._try_cast_result(result)
 
-        return make_block(result, self.items, self.ref_items, ndim=self.ndim, fastpath=True)
+        return [make_block(result, self.items, self.ref_items, ndim=self.ndim, fastpath=True)]
 
     def where(self, other, cond, raise_on_error=True, try_cast=False):
         """
@@ -1054,6 +1061,14 @@ def _try_fill(self, value):
             value = tslib.iNaT
         return value
 
+    def fillna(self, value, inplace=False, downcast=None):
+        values = self.values if inplace else self.values.copy()
+        mask = com.isnull(self.values)
+        value = self._try_fill(value)
+        np.putmask(values,mask,value)
+        return [self if inplace else make_block(values, self.items,
+                                                self.ref_items, fastpath=True)]
+
     def to_native_types(self, slicer=None, na_rep=None, **kwargs):
         """ convert to our native types format, slicing if desired """
 
@@ -1250,7 +1265,7 @@ def fillna(self, value, inplace=False, downcast=None):
         if issubclass(self.dtype.type, np.floating):
             value = float(value)
         values = self.values if inplace else self.values.copy()
-        return self.make_block(values.get_values(value), fill_value=value)
+        return [ self.make_block(values.get_values(value), fill_value=value) ]
 
     def shift(self, indexer, periods):
         """ shift the block by periods """
@@ -1263,15 +1278,15 @@ def shift(self, indexer, periods):
             new_values[:periods] = fill_value
         else:
             new_values[periods:] = fill_value
-        return self.make_block(new_values)
+        return [ self.make_block(new_values) ]
 
     def take(self, indexer, ref_items, axis=1):
         """ going to take our items
             along the long dimension"""
         if axis < 1:
             raise AssertionError('axis must be at least 1, got %d' % axis)
 
-        return self.make_block(self.values.take(indexer))
+        return [ self.make_block(self.values.take(indexer)) ]
 
     def reindex_axis(self, indexer, method=None, axis=1, fill_value=None, limit=None, mask_info=None):
         """

pandas/core/series.py

@@ -2687,108 +2687,6 @@ def apply(self, func, convert_dtype=True, args=(), **kwds):
         else:
             return self._constructor(mapped, index=self.index, name=self.name)
 
-    def replace(self, to_replace, value=None, method='pad', inplace=False,
-                limit=None):
-        """
-        Replace arbitrary values in a Series
-
-        Parameters
-        ----------
-        to_replace : list or dict
-            list of values to be replaced or dict of replacement values
-        value : anything
-            if to_replace is a list then value is the replacement value
-        method : {'backfill', 'bfill', 'pad', 'ffill', None}, default 'pad'
-            Method to use for filling holes in reindexed Series
-            pad / ffill: propagate last valid observation forward to next valid
-            backfill / bfill: use NEXT valid observation to fill gap
-        inplace : boolean, default False
-            If True, fill the Series in place. Note: this will modify any other
-            views on this Series, for example a column in a DataFrame. Returns
-            a reference to the filled object, which is self if inplace=True
-        limit : int, default None
-            Maximum size gap to forward or backward fill
-
-        Notes
-        -----
-        replace does not distinguish between NaN and None
-
-        See also
-        --------
-        fillna, reindex, asfreq
-
-        Returns
-        -------
-        replaced : Series
-        """
-
-        if inplace:
-            result = self
-            change = self
-        else:
-            result = self.copy()
-            change = None
-
-        def _rep_one(s, to_rep, v):  # replace single value
-            mask = com.mask_missing(s.values, to_rep)
-            com._maybe_upcast_putmask(s.values, mask, v, change=change)
-
-        def _rep_dict(rs, to_rep):  # replace {[src] -> dest}
-
-            all_src = set()
-            dd = {}  # group by unique destination value
-            for s, d in to_rep.iteritems():
-                dd.setdefault(d, []).append(s)
-                all_src.add(s)
-
-            if any(d in all_src for d in dd.keys()):
-                # don't clobber each other at the cost of temporaries
-                masks = {}
-                for d, sset in dd.iteritems():  # now replace by each dest
-                    masks[d] = com.mask_missing(rs.values, sset)
-
-                for d, m in masks.iteritems():
-                    com._maybe_upcast_putmask(rs.values, m, d, change=change)
-            else:  # if no risk of clobbering then simple
-                for d, sset in dd.iteritems():
-                    _rep_one(rs, sset, d)
-
-        if np.isscalar(to_replace):
-            to_replace = [to_replace]
-
-        if isinstance(to_replace, dict):
-            _rep_dict(result, to_replace)
-        elif isinstance(to_replace, (list, pa.Array, Series)):
-
-            # check same length
-            if isinstance(value, (list, pa.Array, Series)):
-                vl, rl = len(value), len(to_replace)
-                if vl == rl:
-                    _rep_dict(result, dict(zip(to_replace, value)))
-                else:
-                    raise ValueError('Got %d to replace but %d values'
-                                     % (rl, vl))
-
-            elif value is not None:  # otherwise all replaced with same value
-                _rep_one(result, to_replace, value)
-            else:  # method
-                if method is None:  # pragma: no cover
-                    raise ValueError('must specify a fill method')
-                fill_f = _get_fill_func(method)
-
-                mask = com.mask_missing(result.values, to_replace)
-                fill_f(result.values, limit=limit, mask=mask)
-
-                if not inplace:
-                    result = Series(result.values, index=self.index,
-                                    name=self.name)
-        else:
-            raise ValueError('Unrecognized to_replace type %s' %
-                             type(to_replace))
-
-        if not inplace:
-            return result
-
     def align(self, other, join='outer', level=None, copy=True,
               fill_value=None, method=None, limit=None):
         """

pandas/io/pytables.py

@@ -1430,7 +1430,7 @@ def get_atom_string(self, block, itemsize):
 
     def set_atom_string(self, block, existing_col, min_itemsize, nan_rep, encoding):
         # fill nan items with myself
-        block = block.fillna(nan_rep)
+        block = block.fillna(nan_rep)[0]
         data  = block.values
 
         # see if we have a valid string type

pandas/tests/test_series.py

@@ -4338,16 +4338,6 @@ def test_replace(self):
         rs2 = ser.replace([np.nan, 'foo', 'bar'], [-1, -2, -3])
         assert_series_equal(rs, rs2)
 
-        # replace with forward fill not considering np.nan missing
-        s2 = ser.copy()
-        s2[5] = np.nan
-        rs3 = s2.replace(['foo', 'bar'])
-        self.assert_(isnull(rs3[6]))
-
-        # replace with back fill considering np.nan as missing
-        rs4 = ser.replace([np.nan, 'foo', 'bar'], method='bfill')
-        assert_almost_equal(rs4[4], ser[5])
-
         # replace inplace
         ser.replace([np.nan, 'foo', 'bar'], -1, inplace=True)
 
@@ -4369,7 +4359,7 @@ def test_replace(self):
 
         # malformed
         self.assertRaises(ValueError, ser.replace, [1, 2, 3], [np.nan, 0])
-        self.assertRaises(ValueError, ser.replace, range(1, 3), [np.nan, 0])
+        self.assertRaises(TypeError, ser.replace, range(1, 3), [np.nan, 0])
 
         ser = Series([0, 1, 2, 3, 4])
         result = ser.replace([0, 1, 2, 3, 4], [4, 3, 2, 1, 0])
@@ -4687,16 +4677,6 @@ def test_replace(self):
         rs2 = ser.replace([np.nan, 'foo', 'bar'], [-1, -2, -3])
         assert_series_equal(rs, rs2)
 
-        # replace with forward fill not considering np.nan missing
-        s2 = ser.copy()
-        s2[5] = np.nan
-        rs3 = s2.replace(['foo', 'bar'])
-        self.assert_(isnull(rs3[6]))
-
-        # replace with back fill considering np.nan as missing
-        rs4 = ser.replace([np.nan, 'foo', 'bar'], method='bfill')
-        assert_almost_equal(rs4[4], ser[5])
-
         # replace inplace
         ser.replace([np.nan, 'foo', 'bar'], -1, inplace=True)
         self.assert_((ser[:5] == -1).all())