ENH: fill_value arg in DataFrame.reindex/reindex_axis, add fillna to sparse objects, GH #784

Author: wesm

pandas/core/common.py

@@ -108,15 +108,15 @@ def _take_1d_bool(arr, indexer, out, fill_value=np.nan):
     outview = out.view(np.uint8)
     lib.take_1d_bool(view, indexer, outview, fill_value=fill_value)
 
-def _take_2d_axis0_bool(arr, indexer, out):
+def _take_2d_axis0_bool(arr, indexer, out, fill_value=np.nan):
     view = arr.view(np.uint8)
     outview = out.view(np.uint8)
-    lib.take_2d_axis0_bool(view, indexer, outview)
+    lib.take_2d_axis0_bool(view, indexer, outview, fill_value=fill_value)
 
-def _take_2d_axis1_bool(arr, indexer, out):
+def _take_2d_axis1_bool(arr, indexer, out, fill_value=np.nan):
     view = arr.view(np.uint8)
     outview = out.view(np.uint8)
-    lib.take_2d_axis1_bool(view, indexer, outview)
+    lib.take_2d_axis1_bool(view, indexer, outview, fill_value=fill_value)
 
 _take1d_dict = {
     'float64' : lib.take_1d_float64,
@@ -198,7 +198,8 @@ def take_1d(arr, indexer, out=None, fill_value=np.nan):
 
     return out
 
-def take_2d(arr, indexer, out=None, mask=None, needs_masking=None, axis=0):
+def take_2d(arr, indexer, out=None, mask=None, needs_masking=None, axis=0,
+            fill_value=np.nan):
     """
     Specialized Cython take which sets NaN values in one pass
     """
@@ -221,19 +222,20 @@ def take_2d(arr, indexer, out=None, mask=None, needs_masking=None, axis=0):
             # upcasting may be required
             result = arr.take(indexer, axis=axis, out=out)
             result = _maybe_mask(result, mask, needs_masking, axis=axis,
-                                 out_passed=out is not None)
+                                 out_passed=out is not None,
+                                 fill_value=fill_value)
             return result
         else:
             if out is None:
                 out = np.empty(out_shape, dtype=arr.dtype)
             take_f = _get_take2d_function(dtype_str, axis=axis)
-            take_f(arr, indexer, out=out)
+            take_f(arr, indexer, out=out, fill_value=fill_value)
             return out
     elif dtype_str in ('float64', 'object'):
         if out is None:
             out = np.empty(out_shape, dtype=arr.dtype)
         take_f = _get_take2d_function(dtype_str, axis=axis)
-        take_f(arr, indexer, out=out)
+        take_f(arr, indexer, out=out, fill_value=fill_value)
         return out
     else:
         if mask is None:
@@ -246,34 +248,37 @@ def take_2d(arr, indexer, out=None, mask=None, needs_masking=None, axis=0):
 
         result = arr.take(indexer, axis=axis, out=out)
         result = _maybe_mask(result, mask, needs_masking, axis=axis,
-                             out_passed=out is not None)
+                             out_passed=out is not None,
+                             fill_value=fill_value)
         return result
 
-def null_out_axis(arr, mask, axis):
+def mask_out_axis(arr, mask, axis, fill_value=np.nan):
     indexer = [slice(None)] * arr.ndim
     indexer[axis] = mask
 
-    arr[tuple(indexer)] = np.NaN
+    arr[tuple(indexer)] = fill_value
 
-def take_fast(arr, indexer, mask, needs_masking, axis=0, out=None):
+def take_fast(arr, indexer, mask, needs_masking, axis=0, out=None,
+              fill_value=np.nan):
     if arr.ndim == 2:
         return take_2d(arr, indexer, out=out, mask=mask,
                        needs_masking=needs_masking,
-                       axis=axis)
+                       axis=axis, fill_value=fill_value)
 
     result = arr.take(indexer, axis=axis, out=out)
     result = _maybe_mask(result, mask, needs_masking, axis=axis,
-                         out_passed=out is not None)
+                         out_passed=out is not None, fill_value=fill_value)
     return result
 
-def _maybe_mask(result, mask, needs_masking, axis=0, out_passed=False):
+def _maybe_mask(result, mask, needs_masking, axis=0, out_passed=False,
+                fill_value=np.nan):
     if needs_masking:
         if out_passed and _need_upcast(result):
             raise Exception('incompatible type for NAs')
         else:
             # a bit spaghettified
             result = _maybe_upcast(result)
-            null_out_axis(result, mask, axis)
+            mask_out_axis(result, mask, axis, fill_value)
     return result
 
 def _maybe_upcast(values):

pandas/core/frame.py

@@ -1693,7 +1693,7 @@ def lookup(self, row_labels, col_labels):
     # Reindexing and alignment
 
     def align(self, other, join='outer', axis=None, level=None, copy=True,
-              fill_value=None, method=None):
+              fill_value=np.nan, method=None):
         """
         Align two DataFrame object on their index and columns with the
         specified join method for each axis Index
@@ -1710,7 +1710,9 @@ def align(self, other, join='outer', axis=None, level=None, copy=True,
         copy : boolean, default True
             Always returns new objects. If copy=False and no reindexing is
             required then original objects are returned.
-        fill_value : object, default None
+        fill_value : scalar, default np.NaN
+            Value to use for missing values. Defaults to NaN, but can be any
+            "compatible" value
         method : str, default None
 
         Returns
@@ -1730,7 +1732,7 @@ def align(self, other, join='outer', axis=None, level=None, copy=True,
             raise TypeError('unsupported type: %s' % type(other))
 
     def _align_frame(self, other, join='outer', axis=None, level=None,
-                     copy=True, fill_value=None, method=None):
+                     copy=True, fill_value=np.nan, method=None):
         # defaults
         join_index, join_columns = None, None
         ilidx, iridx = None, None
@@ -1749,15 +1751,17 @@ def _align_frame(self, other, join='outer', axis=None, level=None,
                                       return_indexers=True)
 
         left = self._reindex_with_indexers(join_index, ilidx,
-                                           join_columns, clidx, copy)
+                                           join_columns, clidx, copy,
+                                           fill_value=fill_value)
         right = other._reindex_with_indexers(join_index, iridx,
-                                             join_columns, cridx, copy)
-        fill_na = (fill_value is not None) or (method is not None)
-        if fill_na:
-            return (left.fillna(fill_value, method=method),
-                    right.fillna(fill_value, method=method))
-        else:
-            return left, right
+                                             join_columns, cridx, copy,
+                                             fill_value=fill_value)
+
+        if method is not None:
+            left = left.fillna(method=method)
+            right = right.fillna(method=method)
+
+        return left, right
 
     def _align_series(self, other, join='outer', axis=None, level=None,
                       copy=True, fill_value=None, method=None):
@@ -1798,7 +1802,7 @@ def _align_series(self, other, join='outer', axis=None, level=None,
             return left_result, right_result
 
     def reindex(self, index=None, columns=None, method=None, level=None,
-                copy=True):
+                fill_value=np.nan, copy=True):
         """Conform DataFrame to new index with optional filling logic, placing
         NA/NaN in locations having no value in the previous index. A new object
         is produced unless the new index is equivalent to the current one and
@@ -1820,6 +1824,9 @@ def reindex(self, index=None, columns=None, method=None, level=None,
         level : int or name
             Broadcast across a level, matching Index values on the
             passed MultiIndex level
+        fill_value : scalar, default np.NaN
+            Value to use for missing values. Defaults to NaN, but can be any
+            "compatible" value
 
         Examples
         --------
@@ -1833,14 +1840,15 @@ def reindex(self, index=None, columns=None, method=None, level=None,
         frame = self
 
         if index is not None:
-            frame = frame._reindex_index(index, method, copy, level)
+            frame = frame._reindex_index(index, method, copy, level, fill_value)
 
         if columns is not None:
-            frame = frame._reindex_columns(columns, copy, level)
+            frame = frame._reindex_columns(columns, copy, level, fill_value)
 
         return frame
 
-    def reindex_axis(self, labels, axis=0, method=None, level=None, copy=True):
+    def reindex_axis(self, labels, axis=0, method=None, level=None, copy=True,
+                     fill_value=np.nan):
         """Conform DataFrame to new index with optional filling logic, placing
         NA/NaN in locations having no value in the previous index. A new object
         is produced unless the new index is equivalent to the current one and
@@ -1878,42 +1886,45 @@ def reindex_axis(self, labels, axis=0, method=None, level=None, copy=True):
         """
         self._consolidate_inplace()
         if axis == 0:
-            df = self._reindex_index(labels, method, copy, level)
-            return df
+            return self._reindex_index(labels, method, copy, level,
+                                       fill_value=fill_value)
         elif axis == 1:
-            df = self._reindex_columns(labels, copy, level)
-            return df
+            return self._reindex_columns(labels, copy, level,
+                                         fill_value=fill_value)
         else:  # pragma: no cover
             raise ValueError('Must specify axis=0 or 1')
 
-    def _reindex_index(self, new_index, method, copy, level):
+    def _reindex_index(self, new_index, method, copy, level, fill_value=np.nan):
         if level is not None:
             assert(isinstance(new_index, MultiIndex))
         new_index, indexer = self.index.reindex(new_index, method, level)
         return self._reindex_with_indexers(new_index, indexer, None, None,
-                                           copy)
+                                           copy, fill_value)
 
-    def _reindex_columns(self, new_columns, copy, level):
+    def _reindex_columns(self, new_columns, copy, level, fill_value=np.nan):
         if level is not None:
             assert(isinstance(new_columns, MultiIndex))
         new_columns, indexer = self.columns.reindex(new_columns, level=level)
         return self._reindex_with_indexers(None, None, new_columns, indexer,
-                                           copy)
+                                           copy, fill_value)
 
     def _reindex_with_indexers(self, index, row_indexer, columns, col_indexer,
-                               copy):
+                               copy, fill_value):
         new_data = self._data
         if row_indexer is not None:
-            new_data = new_data.reindex_indexer(index, row_indexer, axis=1)
+            new_data = new_data.reindex_indexer(index, row_indexer, axis=1,
+                                                fill_value=fill_value)
         elif index is not None and index is not new_data.axes[1]:
             new_data = new_data.copy(deep=copy)
             new_data.axes[1] = index
 
         if col_indexer is not None:
             # TODO: speed up on homogeneous DataFrame objects
-            new_data = new_data.reindex_indexer(columns, col_indexer, axis=0)
+            new_data = new_data.reindex_indexer(columns, col_indexer, axis=0,
+                                                fill_value=fill_value)
         elif columns is not None and columns is not new_data.axes[0]:
-            new_data = new_data.reindex_items(columns, copy=copy)
+            new_data = new_data.reindex_items(columns, copy=copy,
+                                              fill_value=fill_value)
 
         if copy and new_data is self._data:
             new_data = new_data.copy()
@@ -2361,8 +2372,7 @@ def reorder_levels(self, order, axis=0):
 
     def fillna(self, value=None, method='pad', inplace=False):
         """
-        Fill NA/NaN values using the specified method. Member Series /
-        TimeSeries are filled separately
+        Fill NA/NaN values using the specified method
 
         Parameters
         ----------
@@ -2402,12 +2412,6 @@ def fillna(self, value=None, method='pad', inplace=False):
                 new_blocks.append(newb)
 
             new_data = BlockManager(new_blocks, self._data.axes)
-
-            # series = self._series
-            # for col, s in series.iteritems():
-            #     result[col] = s.fillna(method=method, value=value)
-            # return self._constructor(result, index=self.index,
-            #                          columns=self.columns)
         else:
             # Float type values
             if len(self.columns) == 0:

pandas/core/internals.py

@@ -101,18 +101,20 @@ def merge(self, other):
         #     union_ref = self.ref_items + other.ref_items
         return _merge_blocks([self, other], self.ref_items)
 
-    def reindex_axis(self, indexer, mask, needs_masking, axis=0):
+    def reindex_axis(self, indexer, mask, needs_masking, axis=0,
+                     fill_value=np.nan):
         """
         Reindex using pre-computed indexer information
         """
         if self.values.size > 0:
             new_values = com.take_fast(self.values, indexer, mask,
-                                       needs_masking, axis=axis)
+                                       needs_masking, axis=axis,
+                                       fill_value=fill_value)
         else:
             shape = list(self.shape)
             shape[axis] = len(indexer)
             new_values = np.empty(shape)
-            new_values.fill(np.nan)
+            new_values.fill(fill_value)
         return make_block(new_values, self.items, self.ref_items)
 
     def reindex_items_from(self, new_ref_items, copy=True):
@@ -730,12 +732,12 @@ def reindex_axis(self, new_axis, method=None, axis=0, copy=True):
         new_axis, indexer = cur_axis.reindex(new_axis, method)
         return self.reindex_indexer(new_axis, indexer, axis=axis)
 
-    def reindex_indexer(self, new_axis, indexer, axis=1):
+    def reindex_indexer(self, new_axis, indexer, axis=1, fill_value=np.nan):
         """
         pandas-indexer with -1's only.
         """
         if axis == 0:
-            return self._reindex_indexer_items(new_axis, indexer)
+            return self._reindex_indexer_items(new_axis, indexer, fill_value)
 
         mask = indexer == -1
 
@@ -745,14 +747,14 @@ def reindex_indexer(self, new_axis, indexer, axis=1):
         new_blocks = []
         for block in self.blocks:
             newb = block.reindex_axis(indexer, mask, needs_masking,
-                                      axis=axis)
+                                      axis=axis, fill_value=fill_value)
             new_blocks.append(newb)
 
         new_axes = list(self.axes)
         new_axes[axis] = new_axis
         return BlockManager(new_blocks, new_axes)
 
-    def _reindex_indexer_items(self, new_items, indexer):
+    def _reindex_indexer_items(self, new_items, indexer, fill_value):
         # TODO: less efficient than I'd like
 
         item_order = com.take_1d(self.items.values, indexer)
@@ -778,13 +780,14 @@ def _reindex_indexer_items(self, new_items, indexer):
 
         if not mask.all():
             na_items = new_items[-mask]
-            na_block = self._make_na_block(na_items, new_items)
+            na_block = self._make_na_block(na_items, new_items,
+                                           fill_value=fill_value)
             new_blocks.append(na_block)
             new_blocks = _consolidate(new_blocks, new_items)
 
         return BlockManager(new_blocks, [new_items] + self.axes[1:])
 
-    def reindex_items(self, new_items, copy=True):
+    def reindex_items(self, new_items, copy=True, fill_value=np.nan):
         """
 
         """
@@ -814,17 +817,22 @@ def reindex_items(self, new_items, copy=True):
             mask = indexer == -1
             if mask.any():
                 extra_items = new_items[mask]
-                na_block = self._make_na_block(extra_items, new_items)
+                na_block = self._make_na_block(extra_items, new_items,
+                                               fill_value=fill_value)
                 new_blocks.append(na_block)
                 new_blocks = _consolidate(new_blocks, new_items)
 
         return BlockManager(new_blocks, [new_items] + self.axes[1:])
 
-    def _make_na_block(self, items, ref_items):
+    def _make_na_block(self, items, ref_items, fill_value=np.nan):
+        # TODO: infer dtypes other than float64 from fill_value
+
         block_shape = list(self.shape)
         block_shape[0] = len(items)
-        block_values = np.empty(block_shape, dtype=np.float64)
-        block_values.fill(nan)
+
+        dtype = com._infer_dtype(fill_value)
+        block_values = np.empty(block_shape, dtype=dtype)
+        block_values.fill(fill_value)
         na_block = make_block(block_values, items, ref_items,
                               do_integrity_check=True)
         return na_block