ENH: added ignore_index option to DataFrame.append, and speed optimizations with _ensure_index function

Author: wesm

RELEASE.rst

@@ -17,6 +17,10 @@ This is a bug fix release
 
   -
 
+**Improvements to existing features**
+
+  - Some speed enhancements with internal Index type-checking function
+
 pandas 0.4
 ==========
 

pandas/core/common.py

@@ -138,13 +138,6 @@ def _mut_exclusive(arg1, arg2):
     else:
         return arg2
 
-def _ensure_index(index_like):
-    from pandas.core.index import Index
-    if not isinstance(index_like, Index):
-        index_like = Index(index_like)
-
-    return index_like
-
 def _any_none(*args):
     for arg in args:
         if arg is None:

pandas/core/frame.py

@@ -19,15 +19,15 @@
 from numpy import nan
 import numpy as np
 
-from pandas.core.common import (isnull, notnull, PandasError, _ensure_index,
+from pandas.core.common import (isnull, notnull, PandasError,
                                 _try_sort, _pfixed, _default_index,
                                 _infer_dtype, _stringify)
 from pandas.core.daterange import DateRange
 from pandas.core.generic import AxisProperty, NDFrame
-from pandas.core.index import Index, MultiIndex, NULL_INDEX
+from pandas.core.index import Index, MultiIndex, NULL_INDEX, _ensure_index
 from pandas.core.indexing import _NDFrameIndexer, _maybe_droplevels
 from pandas.core.internals import BlockManager, make_block, form_blocks
-from pandas.core.series import Series, _is_bool_indexer
+from pandas.core.series import Series, _is_bool_indexer, _maybe_upcast
 from pandas.util.decorators import deprecate
 import pandas.core.common as common
 import pandas.core.datetools as datetools
@@ -2008,11 +2008,18 @@ def f(x):
     #----------------------------------------------------------------------
     # Merging / joining methods
 
-    def append(self, other):
+    def append(self, other, ignore_index=False):
         """
         Append columns of other to end of this frame's columns and index.
         Columns not in this frame are added as new columns.
 
+        Parameters
+        ----------
+        other : DataFrame
+        ignore_index : boolean, default False
+            If True do not use the index labels. Useful for gluing together
+            record arrays
+
         Returns
         -------
         appended : DataFrame
@@ -2022,28 +2029,53 @@ def append(self, other):
         if not self:
             return other.copy()
 
-        new_index = np.concatenate((self.index, other.index))
-        new_data = {}
+        if ignore_index:
+            new_index = None
+        else:
+            new_index = np.concatenate((self.index, other.index))
+
+        if self.columns.equals(other.columns):
+            return self._append_same_columns(other, new_index)
+        else:
+            return self._append_different_columns(other, new_index)
 
-        new_columns = self.columns
+    def _append_different_columns(self, other, new_index):
+        new_columns = self.columns + other.columns
+        new_data = self._append_column_by_column(other)
+        return self._constructor(data=new_data, index=new_index,
+                                 columns=new_columns)
+
+    def _append_same_columns(self, other, new_index):
+        if self._is_mixed_type:
+            new_data = self._append_column_by_column(other)
+        else:
+            new_data= np.concatenate((self.values, other.values), axis=0)
+        return self._constructor(new_data, index=new_index,
+                                 columns=self.columns)
 
-        if not new_columns.equals(other.columns):
-            new_columns = self.columns + other.columns
+    def _append_column_by_column(self, other):
+        def _concat_missing(values, n):
+            values = _maybe_upcast(values)
+            missing_values = np.empty(n, dtype=values.dtype)
+            missing_values.fill(np.nan)
+            return values, missing_values
 
-        for column, series in self.iteritems():
-            values = series.values
-            if column in other:
-                other_values = other[column].values
-                new_data[column] = np.concatenate((values, other_values))
+        new_data = {}
+        for col in self:
+            values = self._data.get(col)
+            if col in other:
+                other_values = other._data.get(col)
             else:
-                new_data[column] = series
+                values, other_values = _concat_missing(values, len(other))
+            new_data[col] = np.concatenate((values, other_values))
 
-        for column, series in other.iteritems():
-            if column not in self:
-                new_data[column] = series
+        for col in other:
+            values = other._data.get(col)
+            if col not in self:
+                values, missing_values = _concat_missing(values, len(self))
+                new_data[col] = np.concatenate((missing_values, values))
 
-        return self._constructor(data=new_data, index=new_index,
-                                 columns=new_columns)
+        return new_data
 
     def join(self, other, on=None, how=None, lsuffix='', rsuffix=''):
         """
@@ -3137,7 +3169,6 @@ def _homogenize(data, index, columns, dtype=None):
 
     return homogenized
 
-
 def _put_str(s, space):
     return ('%s' % s)[:space].ljust(space)
 

pandas/core/generic.py

@@ -1,8 +1,7 @@
 import numpy as np
 import cPickle
 
-from pandas.core.common import _ensure_index
-from pandas.core.index import Index, MultiIndex
+from pandas.core.index import Index, MultiIndex, _ensure_index
 import pandas.core.datetools as datetools
 
 #-------------------------------------------------------------------------------

pandas/core/index.py

@@ -6,8 +6,7 @@
 import numpy as np
 
 from pandas.core.common import (_format, adjoin as _adjoin, _stringify,
-                                _ensure_index, _is_bool_indexer,
-                                _asarray_tuplesafe)
+                                _is_bool_indexer, _asarray_tuplesafe)
 from pandas.util.decorators import deprecate, cache_readonly
 import pandas.core.common as common
 import pandas._tseries as _tseries
@@ -1226,3 +1225,7 @@ def _sparsify(label_list):
 
     return zip(*result)
 
+def _ensure_index(index_like):
+    if isinstance(index_like, Index):
+        return index_like
+    return Index(index_like)

pandas/core/internals.py

@@ -3,8 +3,7 @@
 from numpy import nan
 import numpy as np
 
-from pandas.core.index import Index
-from pandas.core.common import _ensure_index
+from pandas.core.index import Index, _ensure_index
 import pandas.core.common as common
 import pandas._tseries as _tseries
 

pandas/core/panel.py

@@ -10,9 +10,9 @@
 
 import numpy as np
 
-from pandas.core.common import (PandasError, _mut_exclusive, _ensure_index,
+from pandas.core.common import (PandasError, _mut_exclusive,
                                 _try_sort, _default_index, _infer_dtype)
-from pandas.core.index import Factor, Index, MultiIndex
+from pandas.core.index import Factor, Index, MultiIndex, _ensure_index
 from pandas.core.indexing import _NDFrameIndexer
 from pandas.core.internals import BlockManager, make_block, form_blocks
 from pandas.core.frame import DataFrame, _union_indexes

pandas/core/series.py

@@ -13,11 +13,11 @@
 from numpy import nan, ndarray
 import numpy as np
 
-from pandas.core.common import (isnull, notnull, _ensure_index,
-                                _is_bool_indexer, _default_index)
+from pandas.core.common import (isnull, notnull, _is_bool_indexer,
+                                _default_index)
 from pandas.core.daterange import DateRange
 from pandas.core.generic import PandasObject
-from pandas.core.index import Index, MultiIndex
+from pandas.core.index import Index, MultiIndex, _ensure_index
 from pandas.core.indexing import _SeriesIndexer, _maybe_droplevels
 from pandas.util.decorators import deprecate
 import pandas.core.datetools as datetools

pandas/core/sparse.py

@@ -11,8 +11,8 @@
 import operator
 
 from pandas.core.common import (isnull, _pickle_array, _unpickle_array,
-                                _mut_exclusive, _ensure_index, _try_sort)
-from pandas.core.index import Index, MultiIndex, NULL_INDEX
+                                _mut_exclusive, _try_sort)
+from pandas.core.index import Index, MultiIndex, NULL_INDEX, _ensure_index
 from pandas.core.series import Series, TimeSeries
 from pandas.core.frame import (DataFrame, extract_index, _prep_ndarray,
                                _default_index)
@@ -624,7 +624,7 @@ class SparseDataFrame(DataFrame):
     _verbose_info = False
     _columns = None
     _series = None
-
+    _is_mixed_type = False
     ndim = 2
 
     def __init__(self, data=None, index=None, columns=None,
@@ -1056,6 +1056,24 @@ def _rename_columns_inplace(self, mapper):
         self.columns = new_columns
         self._series = new_series
 
+    def _append_column_by_column(self, other):
+        new_data = {}
+        for col in self:
+            values = self[col].values
+            if col in other:
+                other_values = other[col].values
+            else:
+                values = _maybe_upcast(values)
+                other_values = np.empty(len(other), dtype=values.dtype)
+                other_values.fill(np.nan)
+            new_data[col] = np.concatenate((values, other_values))
+
+        for column, series in other.iteritems():
+            if column not in self:
+                new_data[column] = series.values
+
+        return new_data
+
     def add_prefix(self, prefix):
         f = (('%s' % prefix) + '%s').__mod__
         return self.rename(columns=f)

pandas/tests/test_frame.py

@@ -1517,6 +1517,22 @@ def test_append(self):
         assert_frame_equal(self.frame, appended)
         self.assert_(appended is not self.frame)
 
+    def test_append_records(self):
+        arr1 = np.zeros((2,),dtype=('i4,f4,a10'))
+        arr1[:] = [(1,2.,'Hello'),(2,3.,"World")]
+
+        arr2 = np.zeros((3,),dtype=('i4,f4,a10'))
+        arr2[:] = [(3, 4.,'foo'),
+                   (5, 6.,"bar"),
+                   (7., 8., 'baz')]
+
+        df1 = DataFrame(arr1)
+        df2 = DataFrame(arr2)
+
+        result = df1.append(df2, ignore_index=True)
+        expected = DataFrame(np.concatenate((arr1, arr2)))
+        assert_frame_equal(result, expected)
+
     def test_asfreq(self):
         offset_monthly = self.tsframe.asfreq(datetools.bmonthEnd)
         rule_monthly = self.tsframe.asfreq('EOM')