ENH: optimize fillna for 2D blocks, add inplace option, GH #788, #786

Author: wesm

RELEASE.rst

@@ -22,6 +22,16 @@ Where to get it
 * Binary installers on PyPI: http://pypi.python.org/pypi/pandas
 * Documentation: http://pandas.pydata.org
 
+pandas 0.7.1
+============
+
+**Release date:** NOT YET RELEASED
+
+**Bug fixes**
+
+  - Fix memory leak when inserting large number of columns into a single
+    DataFrame (#790)
+
 pandas 0.7.0
 ============
 

pandas/core/common.py

@@ -562,6 +562,13 @@ def _ensure_object(arr):
         arr = arr.astype('O')
     return arr
 
+def _clean_fill_method(method):
+    method = method.lower()
+    if method == 'ffill':
+        method = 'pad'
+    if method == 'bfill':
+        method = 'backfill'
+    return method
 
 def save(obj, path):
     """

pandas/core/frame.py

@@ -2307,7 +2307,7 @@ def reorder_levels(self, order, axis=0):
     #----------------------------------------------------------------------
     # Filling NA's
 
-    def fillna(self, value=None, method='pad'):
+    def fillna(self, value=None, method='pad', inplace=False):
         """
         Fill NA/NaN values using the specified method. Member Series /
         TimeSeries are filled separately
@@ -2320,6 +2320,11 @@ def fillna(self, value=None, method='pad'):
             backfill / bfill: use NEXT valid observation to fill gap
         value : any kind (should be same type as array)
             Value to use to fill holes (e.g. 0)
+        inplace : boolean, default False
+            If True, fill the DataFrame in place. Note: this will modify any
+            other views on this DataFrame, like if you took a no-copy slice of
+            an existing DataFrame, for example a column in a DataFrame. Returns
+            a reference to the filled object, which is self if inplace=True
 
         See also
         --------
@@ -2329,18 +2334,38 @@ def fillna(self, value=None, method='pad'):
         -------
         filled : DataFrame
         """
+        from pandas.core.internals import FloatBlock, ObjectBlock
+
+        self._consolidate_inplace()
+
         if value is None:
-            result = {}
-            series = self._series
-            for col, s in series.iteritems():
-                result[col] = s.fillna(method=method, value=value)
-            return self._constructor(result, index=self.index,
-                                     columns=self.columns)
+            new_blocks = []
+
+            method = com._clean_fill_method(method)
+            for block in self._data.blocks:
+                if isinstance(block, (FloatBlock, ObjectBlock)):
+                    newb = block.interpolate(method, inplace=inplace)
+                else:
+                    newb = block if inplace else block.copy()
+                new_blocks.append(newb)
+
+            new_data = BlockManager(new_blocks, self._data.axes)
+
+            # series = self._series
+            # for col, s in series.iteritems():
+            #     result[col] = s.fillna(method=method, value=value)
+            # return self._constructor(result, index=self.index,
+            #                          columns=self.columns)
         else:
             # Float type values
             if len(self.columns) == 0:
                 return self
             new_data = self._data.fillna(value)
+
+        if inplace:
+            self._data = new_data
+            return self
+        else:
             return self._constructor(new_data)
 
     #----------------------------------------------------------------------

pandas/core/internals.py

@@ -209,6 +209,39 @@ def fillna(self, value):
         new_values.flat[mask] = value
         return make_block(new_values, self.items, self.ref_items)
 
+    def interpolate(self, method='pad', inplace=False):
+        values = self.values if inplace else self.values.copy()
+
+        if values.ndim != 2:
+            raise NotImplementedError
+
+        if method == 'pad':
+            _pad(values)
+        else:
+            _backfill(values)
+
+        return make_block(values, self.items, self.ref_items)
+
+def _pad(values):
+    if com.is_float_dtype(values):
+        _method = lib.pad_2d_inplace_float64
+    elif values.dtype == np.object_:
+        _method = lib.pad_2d_inplace_object
+    else:
+        raise ValueError('Invalid dtype for padding')
+
+    _method(values, com.isnull(values).view(np.uint8))
+
+def _backfill(values):
+    if com.is_float_dtype(values):
+        _method = lib.backfill_2d_inplace_float64
+    elif values.dtype == np.object_:
+        _method = lib.backfill_2d_inplace_object
+    else:
+        raise ValueError('Invalid dtype for padding')
+
+    _method(values, com.isnull(values).view(np.uint8))
+
 #-------------------------------------------------------------------------------
 # Is this even possible?
 

pandas/core/series.py

@@ -1832,7 +1832,7 @@ def take(self, indices, axis=0):
 
     truncate = generic.truncate
 
-    def fillna(self, value=None, method='pad'):
+    def fillna(self, value=None, method='pad', inplace=False):
         """
         Fill NA/NaN values using the specified method
 
@@ -1844,6 +1844,10 @@ def fillna(self, value=None, method='pad'):
             Method to use for filling holes in reindexed Series
             pad / ffill: propagate last valid observation forward to next valid
             backfill / bfill: use NEXT valid observation to fill gap
+        inplace : boolean, default False
+            If True, fill the Series in place. Note: this will modify any other
+            views on this Series, for example a column in a DataFrame. Returns
+            a reference to the filled object, which is self if inplace=True
 
         See also
         --------
@@ -1853,22 +1857,16 @@ def fillna(self, value=None, method='pad'):
         -------
         filled : Series
         """
+        mask = isnull(self.values)
+
         if value is not None:
-            newSeries = self.copy()
-            newSeries[isnull(newSeries)] = value
-            return newSeries
+            result = self.copy() if not inplace else self
+            np.putmask(result, mask, value)
         else:
             if method is None:  # pragma: no cover
                 raise ValueError('must specify a fill method')
 
-            method = method.lower()
-
-            if method == 'ffill':
-                method = 'pad'
-            if method == 'bfill':
-                method = 'backfill'
-
-            mask = isnull(self.values)
+            method = com._clean_fill_method(method)
 
             # sadness. for Python 2.5 compatibility
             mask = mask.astype(np.uint8)
@@ -1878,8 +1876,14 @@ def fillna(self, value=None, method='pad'):
             elif method == 'backfill':
                 indexer = lib.get_backfill_indexer(mask)
 
-            new_values = self.values.take(indexer)
-            return Series(new_values, index=self.index, name=self.name)
+            if inplace:
+                self.values[:] = self.values.take(indexer)
+                result = self
+            else:
+                new_values = self.values.take(indexer)
+                result = Series(new_values, index=self.index, name=self.name)
+
+        return result
 
     def isin(self, values):
         """

pandas/src/generate_code.py

@@ -268,6 +268,105 @@ def pad_%(name)s(ndarray[%(c_type)s] oldIndex,
 
 """
 
+pad_template = """@cython.boundscheck(False)
+@cython.wraparound(False)
+def pad_%(name)s(ndarray[%(c_type)s] oldIndex,
+                 ndarray[%(c_type)s] newIndex,
+                 dict oldMap, dict newMap):
+    cdef Py_ssize_t i, j, oldLength, newLength, curLoc
+    cdef ndarray[int32_t, ndim=1] fill_vec
+    cdef Py_ssize_t newPos, oldPos
+    cdef %(c_type)s prevOld, curOld
+
+    oldLength = len(oldIndex)
+    newLength = len(newIndex)
+
+    fill_vec = np.empty(len(newIndex), dtype = np.int32)
+    fill_vec.fill(-1)
+
+    if oldLength == 0 or newLength == 0:
+        return fill_vec
+
+    oldPos = 0
+    newPos = 0
+
+    if newIndex[newLength - 1] < oldIndex[0]:
+        return fill_vec
+
+    while newPos < newLength:
+        curOld = oldIndex[oldPos]
+
+        while newIndex[newPos] < curOld:
+            newPos += 1
+            if newPos > newLength - 1:
+                break
+
+        curLoc = oldMap[curOld]
+
+        if oldPos == oldLength - 1:
+            if newIndex[newPos] >= curOld:
+                fill_vec[newPos:] = curLoc
+            break
+        else:
+            nextOld = oldIndex[oldPos + 1]
+            done = 0
+
+            while newIndex[newPos] < nextOld:
+                fill_vec[newPos] = curLoc
+                newPos += 1
+
+                if newPos > newLength - 1:
+                    done = 1
+                    break
+
+            if done:
+                break
+
+        oldPos += 1
+
+    return fill_vec
+
+"""
+
+pad_2d_template = """@cython.boundscheck(False)
+@cython.wraparound(False)
+def pad_2d_inplace_%(name)s(ndarray[%(c_type)s, ndim=2] values,
+                            ndarray[uint8_t, ndim=2] mask):
+    cdef Py_ssize_t i, j, N, K
+    cdef %(c_type)s val
+
+    K, N = (<object> values).shape
+
+    val = np.nan
+
+    for j in range(K):
+        val = values[j, 0]
+        for i in range(N):
+            if mask[j, i]:
+                values[j, i] = val
+            else:
+                val = values[j, i]
+"""
+
+backfill_2d_template = """@cython.boundscheck(False)
+@cython.wraparound(False)
+def backfill_2d_inplace_%(name)s(ndarray[%(c_type)s, ndim=2] values,
+                                 ndarray[uint8_t, ndim=2] mask):
+    cdef Py_ssize_t i, j, N, K
+    cdef %(c_type)s val
+
+    K, N = (<object> values).shape
+
+    for j in range(K):
+        val = values[j, N - 1]
+        for i in range(N - 1, -1 , -1):
+            if mask[j, i]:
+                values[j, i] = val
+            else:
+                val = values[j, i]
+"""
+
+
 is_monotonic_template = """@cython.boundscheck(False)
 @cython.wraparound(False)
 def is_monotonic_%(name)s(ndarray[%(c_type)s] arr):
@@ -638,6 +737,8 @@ def generate_from_template(template, ndim=1, exclude=None):
                 merge_indexer_template,
                 pad_template,
                 backfill_template,
+                pad_2d_template,
+                backfill_2d_template,
                 take_1d_template,
                 is_monotonic_template,
                 groupby_template,