BUG: basic DataFrame constructor refactoring to better support duplicate columns. close #2079

Author: wesm

RELEASE.rst

@@ -55,6 +55,7 @@ pandas 0.9.1
 
 **Bug fixes**
 
+  - Fix some duplicate-column DataFrame constructor issues (#2079)
   - Fix bar plot color cycle issues (#2082)
   - Implement comparisons on date offsets with fixed delta (#2078)
   - Handle inf/-inf correctly in read_* parser functions (#2041)

pandas/core/frame.py

@@ -401,15 +401,14 @@ def __init__(self, data=None, index=None, columns=None, dtype=None,
                     index = _get_names_from_index(data)
 
                 if isinstance(data[0], (list, tuple, dict, Series)):
-                    conv_data, columns = _to_sdict(data, columns)
-                    if isinstance(conv_data, dict):
-                        if len(conv_data) == 0 and index is None:
-                            index = np.arange(len(data))
-                        mgr = self._init_dict(conv_data, index, columns,
-                                              dtype=dtype)
-                    else:
-                        mgr = self._init_ndarray(conv_data, index, columns,
-                                                 dtype=dtype, copy=copy)
+                    arrays, columns = _to_arrays(data, columns)
+
+                    columns = _ensure_index(columns)
+
+                    if index is None:
+                        index = _default_index(len(data))
+                    mgr = self._init_arrays(arrays, columns, index, columns,
+                                            dtype=dtype)
                 else:
                     mgr = self._init_ndarray(data, index, columns, dtype=dtype,
                                              copy=copy)
@@ -463,13 +462,20 @@ def _init_dict(self, data, index, columns, dtype=None):
             index = _ensure_index(index)
 
         # don't force copy because getting jammed in an ndarray anyway
-        homogenized = _homogenize(data, index, columns, dtype)
+        hom_arrays, arr_names = _homogenize(data, index, columns, dtype)
+
+        return self._init_arrays(hom_arrays, arr_names, index, columns)
 
+    def _init_arrays(self, arrays, arr_names, index, columns, dtype=None):
+        """
+        Segregate Series based on type and coerce into matrices.
+        Needs to handle a lot of exceptional cases.
+        """
         # from BlockManager perspective
         axes = [columns, index]
 
         # segregates dtypes and forms blocks matching to columns
-        blocks = form_blocks(homogenized, axes)
+        blocks = form_blocks(arrays, arr_names, axes)
 
         # consolidate for now
         mgr = BlockManager(blocks, axes)
@@ -870,8 +876,9 @@ def from_records(cls, data, index=None, exclude=None, columns=None,
         if isinstance(data, (np.ndarray, DataFrame, dict)):
             columns, sdict = _rec_to_dict(data)
         else:
-            sdict, columns = _to_sdict(data, columns,
-                                       coerce_float=coerce_float)
+            arrays, columns = _to_arrays(data, columns,
+                                         coerce_float=coerce_float)
+            sdict = dict(zip(columns, arrays))
 
         if exclude is None:
             exclude = set()
@@ -5053,9 +5060,13 @@ def _rec_to_dict(arr):
     return columns, sdict
 
 
-def _to_sdict(data, columns, coerce_float=False):
+def _to_arrays(data, columns, coerce_float=False):
+    """
+    Return list of arrays, columns
+    """
+
     if len(data) == 0:
-        return {}, columns
+        return [], columns if columns is not None else []
     if isinstance(data[0], (list, tuple)):
         return _list_to_sdict(data, columns, coerce_float=coerce_float)
     elif isinstance(data[0], dict):
@@ -5100,14 +5111,15 @@ def _list_of_series_to_sdict(data, columns, coerce_float=False):
             indexer = indexer_cache[id(index)] = index.get_indexer(columns)
         aligned_values.append(com.take_1d(s.values, indexer))
 
+    # TODO: waste
     values = np.vstack(aligned_values)
 
     if values.dtype == np.object_:
         content = list(values.T)
         return _convert_object_array(content, columns,
                                      coerce_float=coerce_float)
     else:
-        return values, columns
+        return values.T, columns
 
 
 def _list_of_dict_to_sdict(data, columns, coerce_float=False):
@@ -5133,9 +5145,10 @@ def _convert_object_array(content, columns, coerce_float=False):
             raise AssertionError('%d columns passed, passed data had %s '
                                  'columns' % (len(columns), len(content)))
 
-    sdict = dict((c, lib.maybe_convert_objects(vals, try_float=coerce_float))
-                 for c, vals in zip(columns, content))
-    return sdict, columns
+    arrays = [lib.maybe_convert_objects(arr, try_float=coerce_float)
+              for arr in content]
+
+    return arrays, columns
 
 
 def _get_names_from_index(data):
@@ -5159,13 +5172,14 @@ def _get_names_from_index(data):
 def _homogenize(data, index, columns, dtype=None):
     from pandas.core.series import _sanitize_array
 
-    homogenized = {}
-
     if dtype is not None:
         dtype = np.dtype(dtype)
 
     oindex = None
 
+    homogenized = []
+    names = []
+
     for k in columns:
         if k not in data:
             # no obvious "empty" int column
@@ -5202,9 +5216,10 @@ def _homogenize(data, index, columns, dtype=None):
             v = _sanitize_array(v, index, dtype=dtype, copy=False,
                                 raise_cast_failure=False)
 
-        homogenized[k] = v
+        names.append(k)
+        homogenized.append(v)
 
-    return homogenized
+    return homogenized, names
 
 
 def _put_str(s, space):

pandas/core/internals.py

@@ -17,8 +17,7 @@ class Block(object):
     """
     __slots__ = ['items', 'ref_items', '_ref_locs', 'values', 'ndim']
 
-    def __init__(self, values, items, ref_items, ndim=2,
-                 do_integrity_check=False):
+    def __init__(self, values, items, ref_items, ndim=2):
         if issubclass(values.dtype.type, basestring):
             values = np.array(values, dtype=object)
 
@@ -31,15 +30,6 @@ def __init__(self, values, items, ref_items, ndim=2,
         self.items = _ensure_index(items)
         self.ref_items = _ensure_index(ref_items)
 
-        if do_integrity_check:
-            self._check_integrity()
-
-    def _check_integrity(self):
-        if len(self.items) < 2:
-            return
-        # monotonicity
-        return (self.ref_locs[1:] > self.ref_locs[:-1]).all()
-
     @property
     def ref_locs(self):
         if self._ref_locs is None:
@@ -400,13 +390,11 @@ def should_store(self, value):
 class DatetimeBlock(Block):
     _can_hold_na = True
 
-    def __init__(self, values, items, ref_items, ndim=2,
-                 do_integrity_check=False):
+    def __init__(self, values, items, ref_items, ndim=2):
         if values.dtype != _NS_DTYPE:
             values = lib.cast_to_nanoseconds(values)
 
-        Block.__init__(self, values, items, ref_items, ndim=ndim,
-                       do_integrity_check=do_integrity_check)
+        Block.__init__(self, values, items, ref_items, ndim=ndim)
 
     def _can_hold_element(self, element):
         return com.is_integer(element) or isinstance(element, datetime)
@@ -443,7 +431,7 @@ def get_values(self, dtype):
         return self.values
 
 
-def make_block(values, items, ref_items, do_integrity_check=False):
+def make_block(values, items, ref_items):
     dtype = values.dtype
     vtype = dtype.type
 
@@ -462,8 +450,7 @@ def make_block(values, items, ref_items, do_integrity_check=False):
     else:
         klass = ObjectBlock
 
-    return klass(values, items, ref_items, ndim=values.ndim,
-                 do_integrity_check=do_integrity_check)
+    return klass(values, items, ref_items, ndim=values.ndim)
 
 # TODO: flexible with index=None and/or items=None
 
@@ -548,8 +535,7 @@ def __setstate__(self, state):
 
         blocks = []
         for values, items in zip(bvalues, bitems):
-            blk = make_block(values, items, self.axes[0],
-                             do_integrity_check=True)
+            blk = make_block(values, items, self.axes[0])
             blocks.append(blk)
         self.blocks = blocks
 
@@ -1079,8 +1065,7 @@ def _make_na_block(self, items, ref_items, fill_value=np.nan):
         dtype = com._infer_dtype(fill_value)
         block_values = np.empty(block_shape, dtype=dtype)
         block_values.fill(fill_value)
-        na_block = make_block(block_values, items, ref_items,
-                              do_integrity_check=True)
+        na_block = make_block(block_values, items, ref_items)
         return na_block
 
     def take(self, indexer, axis=1):
@@ -1236,69 +1221,66 @@ def item_dtypes(self):
         assert(mask.all())
         return result
 
-def form_blocks(data, axes):
+def form_blocks(arrays, names, axes):
     # pre-filter out items if we passed it
     items = axes[0]
 
-    if len(data) < len(items):
-        extra_items = items - Index(data.keys())
+    if len(arrays) < len(items):
+        extra_items = items - Index(names)
     else:
         extra_items = []
 
     # put "leftover" items in float bucket, where else?
     # generalize?
-    float_dict = {}
-    complex_dict = {}
-    int_dict = {}
-    bool_dict = {}
-    object_dict = {}
-    datetime_dict = {}
-    for k, v in data.iteritems():
+    float_items = []
+    complex_items = []
+    int_items = []
+    bool_items = []
+    object_items = []
+    datetime_items = []
+    for k, v in zip(names, arrays):
         if issubclass(v.dtype.type, np.floating):
-            float_dict[k] = v
+            float_items.append((k, v))
         elif issubclass(v.dtype.type, np.complexfloating):
-            complex_dict[k] = v
+            complex_items.append((k, v))
         elif issubclass(v.dtype.type, np.datetime64):
             if v.dtype != _NS_DTYPE:
                 v = lib.cast_to_nanoseconds(v)
-            datetime_dict[k] = v
+
+            if hasattr(v, 'tz') and v.tz is not None:
+                object_items.append((k, v))
+            else:
+                datetime_items.append((k, v))
         elif issubclass(v.dtype.type, np.integer):
-            int_dict[k] = v
+            int_items.append((k, v))
         elif v.dtype == np.bool_:
-            bool_dict[k] = v
+            bool_items.append((k, v))
         else:
-            object_dict[k] = v
+            object_items.append((k, v))
 
     blocks = []
-    if len(float_dict):
-        float_block = _simple_blockify(float_dict, items, np.float64)
+    if len(float_items):
+        float_block = _simple_blockify(float_items, items, np.float64)
         blocks.append(float_block)
 
-    if len(complex_dict):
-        complex_block = _simple_blockify(complex_dict, items, np.complex128)
+    if len(complex_items):
+        complex_block = _simple_blockify(complex_items, items, np.complex128)
         blocks.append(complex_block)
 
-    if len(int_dict):
-        int_block = _simple_blockify(int_dict, items, np.int64)
+    if len(int_items):
+        int_block = _simple_blockify(int_items, items, np.int64)
         blocks.append(int_block)
 
-    for k, v in list(datetime_dict.items()):
-        # hackeroo
-        if hasattr(v, 'tz') and v.tz is not None:
-            del datetime_dict[k]
-            object_dict[k] = v.asobject
-
-    if len(datetime_dict):
-        datetime_block = _simple_blockify(datetime_dict, items,
-                                          np.dtype('M8[ns]'))
+    if len(datetime_items):
+        datetime_block = _simple_blockify(datetime_items, items, _NS_DTYPE)
         blocks.append(datetime_block)
 
-    if len(bool_dict):
-        bool_block = _simple_blockify(bool_dict, items, np.bool_)
+    if len(bool_items):
+        bool_block = _simple_blockify(bool_items, items, np.bool_)
         blocks.append(bool_block)
 
-    if len(object_dict) > 0:
-        object_block = _simple_blockify(object_dict, items, np.object_)
+    if len(object_items) > 0:
+        object_block = _simple_blockify(object_items, items, np.object_)
         blocks.append(object_block)
 
     if len(extra_items):
@@ -1309,22 +1291,21 @@ def form_blocks(data, axes):
 
         block_values.fill(nan)
 
-        na_block = make_block(block_values, extra_items, items,
-                              do_integrity_check=True)
+        na_block = make_block(block_values, extra_items, items)
         blocks.append(na_block)
         blocks = _consolidate(blocks, items)
 
     return blocks
 
-def _simple_blockify(dct, ref_items, dtype):
-    block_items, values = _stack_dict(dct, ref_items, dtype)
+def _simple_blockify(tuples, ref_items, dtype):
+    block_items, values = _stack_arrays(tuples, ref_items, dtype)
     # CHECK DTYPE?
     if values.dtype != dtype: # pragma: no cover
         values = values.astype(dtype)
 
-    return make_block(values, block_items, ref_items, do_integrity_check=True)
+    return make_block(values, block_items, ref_items)
 
-def _stack_dict(dct, ref_items, dtype):
+def _stack_arrays(tuples, ref_items, dtype):
     from pandas.core.series import Series
 
     # fml
@@ -1342,17 +1323,18 @@ def _shape_compat(x):
         else:
             return x.shape
 
+    names, arrays = zip(*tuples)
+
     # index may box values
-    items = ref_items[[x in dct for x in ref_items]]
+    items = ref_items[ref_items.isin(names)]
 
-    first = dct[items[0]]
-    shape = (len(dct),) + _shape_compat(first)
+    first = arrays[0]
+    shape = (len(arrays),) + _shape_compat(first)
 
     stacked = np.empty(shape, dtype=dtype)
-    for i, item in enumerate(items):
-        stacked[i] = _asarray_compat(dct[item])
+    for i, arr in enumerate(arrays):
+        stacked[i] = _asarray_compat(arr)
 
-    # stacked = np.vstack([_asarray_compat(dct[k]) for k in items])
     return items, stacked
 
 def _blocks_to_series_dict(blocks, index=None):
@@ -1419,8 +1401,7 @@ def _merge_blocks(blocks, items):
         return blocks[0]
     new_values = _vstack([b.values for b in blocks])
     new_items = blocks[0].items.append([b.items for b in blocks[1:]])
-    new_block = make_block(new_values, new_items, items,
-                           do_integrity_check=True)
+    new_block = make_block(new_values, new_items, items)
     return new_block.reindex_items_from(items)
 
 def _union_block_items(blocks):