fix categorical and sparse

Author: jorisvandenbossche

pandas/core/internals.py

@@ -312,11 +312,11 @@ def ftype(self):
     def merge(self, other):
         return _merge_blocks([self, other])
 
-    def concat_same_type(self, others):
+    def concat_same_type(self, to_concat):
         """
         Concatenate list of single blocks of the same type.
         """
-        values = np.concatenate([self.values] + [o.values for o in others])
+        values = np.concatenate([blk.values for blk in to_concat])
         return self.make_block_same_class(
             values, placement=slice(0, len(values), 1))
 
@@ -2415,6 +2415,19 @@ def to_native_types(self, slicer=None, na_rep='', quoting=None, **kwargs):
         # we are expected to return a 2-d ndarray
         return values.reshape(1, len(values))
 
+    def concat_same_type(self, to_concat):
+        """
+        Concatenate list of single blocks of the same type.
+        """
+        to_concat = [blk.values for blk in to_concat]
+        values = _concat._concat_categorical(to_concat)
+
+        if is_categorical_dtype(values.dtype):
+            return self.make_block_same_class(
+                values, placement=slice(0, len(values), 1))
+        else:
+            return make_block(values, placement=slice(0, len(values), 1))
+
 
 class DatetimeBlock(DatetimeLikeBlockMixin, Block):
     __slots__ = ()
@@ -2692,20 +2705,18 @@ def shift(self, periods, axis=0, mgr=None):
         return [self.make_block_same_class(new_values,
                                            placement=self.mgr_locs)]
 
-    def concat_same_type(self, others):
+    def concat_same_type(self, to_concat):
         """
         Concatenate list of single blocks of the same type.
         """
-        # can maybe replace
-        # from pandas.core.dtypes.concat._concat_datetimetz ?
-        to_concat = [self.values] + [o.values for o in others]
-
-        if len(set([str(x.dtype) for x in to_concat])) != 1:
-            raise ValueError('to_concat must have the same tz')
+        to_concat = [blk.values for blk in to_concat]
+        values = _concat._concat_datetime(to_concat)
 
-        values = to_concat[0]._concat_same_dtype(to_concat, None)
-        return self.make_block_same_class(
-            values, placement=slice(0, len(values), 1))
+        if is_datetimetz(values):
+            return self.make_block_same_class(
+                values, placement=slice(0, len(values), 1))
+        else:
+            return make_block(values, placement=slice(0, len(values), 1))
 
 
 class SparseBlock(NonConsolidatableMixIn, Block):
@@ -2874,6 +2885,18 @@ def sparse_reindex(self, new_index):
         return self.make_block_same_class(values, sparse_index=new_index,
                                           placement=self.mgr_locs)
 
+    def concat_same_type(self, to_concat):
+        """
+        Concatenate list of single blocks of the same type.
+        """
+        to_concat = [blk.values for blk in to_concat]
+        values = _concat._concat_sparse(to_concat)
+
+        return self.make_block_same_class(
+            values, placement=slice(0, len(values), 1))
+        #else:
+        #    return make_block(values, placement=slice(0, len(values), 1))
+
 
 def make_block(values, placement, klass=None, ndim=None, dtype=None,
                fastpath=False):

pandas/core/reshape/concat.py

@@ -3,13 +3,13 @@
 """
 
 import numpy as np
-from pandas import compat, DataFrame, Series, Index, MultiIndex
+from pandas import compat, DataFrame, Series, Index, MultiIndex, SparseSeries
 from pandas.core.index import (_get_objs_combined_axis,
                                _ensure_index, _get_consensus_names,
                                _all_indexes_same)
 from pandas.core.categorical import (_factorize_from_iterable,
                                      _factorize_from_iterables)
-from pandas.core.internals import concatenate_block_managers
+from pandas.core.internals import concatenate_block_managers, SparseBlock
 from pandas.core import common as com
 from pandas.core.generic import NDFrame
 import pandas.core.dtypes.concat as _concat
@@ -364,16 +364,22 @@ def get_result(self):
             if self.axis == 0:
                 name = com._consensus_name_attr(self.objs)
 
-                # check if all series are of the same block type:
-                blocks = [obj._data.blocks[0] for obj in self.objs]
-                if all([type(b) == type(blocks[0]) for b in blocks[1:]]):
-                    new_block = blocks[0].concat_same_type(blocks[1:])
-                    return (Series(new_block, index=self.new_axes[0],
-                                   name=name, fastpath=True)
-                            .__finalize__(self, method='concat'))
-
                 # concat Series with length to keep dtype as much
                 non_empties = [x for x in self.objs if len(x) > 0]
+
+                # check if all series are of the same block type:
+                if len(non_empties) > 0:
+                    blocks = [obj._data.blocks[0] for obj in non_empties]
+                    if all([type(b) == type(blocks[0]) for b in blocks[1:]]):
+                        new_block = blocks[0].concat_same_type(blocks)
+                        if isinstance(new_block, SparseBlock):
+                            cons = SparseSeries
+                        else:
+                            cons = Series
+                        return (cons(new_block, index=self.new_axes[0],
+                                     name=name, fastpath=True)
+                                .__finalize__(self, method='concat'))
+
                 if len(non_empties) > 0:
                     values = [x._values for x in non_empties]
                 else: