[WIP] API/CLN: Refactor DataFrame.append

pandas/core/frame.py

@@ -6338,45 +6338,152 @@ def append(self, other, ignore_index=False,
         3  3
         4  4
         """
-        if isinstance(other, (Series, dict)):
-            if isinstance(other, dict):
-                other = Series(other)
-            if other.name is None and not ignore_index:
+        kwargs = {
+            'ignore_index': ignore_index,
+            'verify_integrity': verify_integrity,
+            'sort': sort,
+        }
+
+        obj_type = type(other)
+        kwargs['_obj_type'] = obj_type
+        if issubclass(obj_type, dict):
+            return self._append_dict(other, **kwargs)
+        elif issubclass(obj_type, Series):
+            return self._append_series(other, **kwargs)
+        elif issubclass(obj_type, DataFrame):
+            return self._append_frame(other, **kwargs)
+        elif issubclass(obj_type, list):
+
+            try:
+                item_type = type(other[0])
+            except IndexError:  # empty list!
+                return self._append_list_of_frames(other, **kwargs)
+            if not all(isinstance(i, item_type) for i in other[1:]):
+                if issubclass(item_type, (dict, Series, DataFrame)):
+                    raise TypeError("When other is a list, its elements must"
+                                    " be all of the same type")
+                else:
+                    raise TypeError("The value of other must be a"
+                                    " DataFrame or Series/dict-like object,"
+                                    " or list of these")
+            kwargs['_item_type'] = item_type
+
+            if issubclass(item_type, dict):
+                return self._append_list_of_dicts(other, **kwargs)
+            elif issubclass(item_type, Series):
+                return self._append_list_of_series(other, **kwargs)
+            elif issubclass(item_type, DataFrame):
+                return self._append_list_of_frames(other, **kwargs)
+            else:
+                raise TypeError("The value of other must be a"
+                                " DataFrame or Series/dict-like object,"
+                                " or list of these")
+        else:
+            raise TypeError("The value of other must be a"
+                            " DataFrame or Series/dict-like object,"
+                            " or list of these")
+
+    def _append_dict(self, other, *args, **kwargs):
+        return self._append_list_of_dicts([other], *args, **kwargs)
+
+    def _append_series(self, other, *args, **kwargs):
+        return self._append_list_of_series([other], *args, **kwargs)
+
+    def _append_frame(self, other, *args, **kwargs):
+        return self._append_list_of_frames([other], *args, **kwargs)
+
+    def _append_list_of_dicts(self, other, *args, **kwargs):
+        if not kwargs['ignore_index']:
+            raise TypeError('Can only append a dict if ignore_index=True')
+        return self._append_frame(DataFrame(other), *args, **kwargs)
+
+    def _append_list_of_series(self, other, *args, **kwargs):
+        if not kwargs['ignore_index']:
+            if any(series.name is None for series in other):
                 raise TypeError('Can only append a Series if ignore_index=True'
                                 ' or if the Series has a name')
 
-            if other.name is None:
-                index = None
-            else:
-                # other must have the same index name as self, otherwise
-                # index name will be reset
-                index = Index([other.name], name=self.index.name)
+        if len(other) == 1:
+            # manually create DF for performance
+            ser = other[0]
+            df = DataFrame(ser.values.reshape(1, ser.shape[0]),
+                           index=[ser.name], columns=ser.index)
+        else:
+            df = DataFrame(other)
 
-            idx_diff = other.index.difference(self.columns)
-            try:
-                combined_columns = self.columns.append(idx_diff)
-            except TypeError:
-                combined_columns = self.columns.astype(object).append(idx_diff)
-            other = other.reindex(combined_columns, copy=False)
-            other = DataFrame(other.values.reshape((1, len(other))),
-                              index=index,
-                              columns=combined_columns)
-            other = other._convert(datetime=True, timedelta=True)
-            if not self.columns.equals(combined_columns):
-                self = self.reindex(columns=combined_columns)
-        elif isinstance(other, list) and not isinstance(other[0], DataFrame):
-            other = DataFrame(other)
-            if (self.columns.get_indexer(other.columns) >= 0).all():
-                other = other.loc[:, self.columns]
+        return self._append_frame(df, *args, **kwargs)
 
+    def _append_list_of_frames(self, other, *args, **kwargs):
+        ignore_index = kwargs['ignore_index']
+        verify_integrity = kwargs['verify_integrity']
+        sort = kwargs['sort']
+        _obj_type = kwargs['_obj_type']
+        _item_type = kwargs.get('_item_type')
+
+        from pandas.core.indexes.api import _normalize_dataframes
         from pandas.core.reshape.concat import concat
-        if isinstance(other, (list, tuple)):
-            to_concat = [self] + other
-        else:
-            to_concat = [self, other]
-        return concat(to_concat, ignore_index=ignore_index,
-                      verify_integrity=verify_integrity,
-                      sort=sort)
+
+        # sorting behavior when sort=None
+        # TODO: remove when kwarg value change
+        if sort is None:
+            # stabilish desired behavior
+            if _obj_type in (dict, Series):
+                # dict/ser
+
+                sort = False
+                warn = False
+            elif _item_type in (dict, Series):
+                # [dict]/[ser]
+
+                if (self.columns.get_indexer(other[0].columns) >= 0).all():
+                    # self.columns >= other[0].columns
+                    sort = False
+                    warn = False
+                else:
+                    sort = True
+                    types = [df.columns.dtype for df in [self] + other]
+                    common = find_common_type(types)
+                    warn = (common == object)
+            else:
+                # frame/[frame]
+
+                if all(self.columns.equals(df.columns) for df in other):
+                    # all values the same
+                    sort = False
+                    warn = False
+                else:
+                    sort = True
+                    types = [df.columns.dtype for df in [self] + other]
+                    common = find_common_type(types)
+                    warn = (common == object)
+
+            # warn if necessary
+            if warn:
+                from pandas.core.indexes.api import _sort_msg
+                warnings.warn(_sort_msg, FutureWarning)
+
+        # The behavior of concat is a bit problematic as it is. To get around,
+        # we prepare the DataFrames before feeding them into concat.
+        to_concat = [self] + other
+        to_concat_norm = _normalize_dataframes(to_concat, sort=sort)
+        result = concat(to_concat_norm, ignore_index=ignore_index,
+                        verify_integrity=verify_integrity, sort=sort)
+
+        # preserve base DataFrame indexes names
+        # XXX: how will this work with MultiIndex (?)
+        result.columns.name = self.columns.name
+        if not ignore_index:
+            result.index.name = self.index.name
+
+        # Reindexing the columns created an artificial float64 where it
+        # was not needed. We can convert the columns back to the expected
+        # type.
+        if result.shape[0] == 1:
+            base_frame = next(df for df in to_concat_norm if df.shape[0] == 1)
+            dtypes = base_frame.dtypes.to_dict()
+            result = result.astype(dtypes)  # won't work well dups cols
+
+        return result
 
     def join(self, other, on=None, how='left', lsuffix='', rsuffix='',
              sort=False):