BUG: GH3495 change core/format/CSVFormatter.save to allow generic way of dealing

     with columns duplicate or not

Author: jreback

RELEASE.rst

@@ -61,16 +61,15 @@ pandas 0.11.1
   - Fix regression in a DataFrame apply with axis=1, objects were not being converted back
     to base dtypes correctly (GH3480_)
   - Fix issue when storing uint dtypes in an HDFStore. (GH3493_)
-  - Fix assigning a new index to a duplicate index in a DataFrame would fail (GH3468_)
-  - ref_locs support to allow duplicative indices across dtypes (GH3468_)
   - Non-unique index support clarified (GH3468_)
 
-    - Fix assigning a new index to a duplicate index in a DataFrame would fail
+    - Fix assigning a new index to a duplicate index in a DataFrame would fail (GH3468_)
     - Fix construction of a DataFrame with a duplicate index
-    - ref_locs support to allow duplicative indices across dtypes
-      (GH2194_)
+    - ref_locs support to allow duplicative indices across dtypes,
+      allows iget support to always find the index (even across dtypes) (GH2194_)
     - applymap on a DataFrame with a non-unique index now works
       (removed warning) (GH2786_), and fix (GH3230_)
+    - Fix to_csv to handle non-unique columns (GH3495_)
 
 .. _GH3164: https://github.com/pydata/pandas/issues/3164
 .. _GH2786: https://github.com/pydata/pandas/issues/2786
@@ -91,6 +90,7 @@ pandas 0.11.1
 .. _GH3468: https://github.com/pydata/pandas/issues/3468
 .. _GH3448: https://github.com/pydata/pandas/issues/3448
 .. _GH3449: https://github.com/pydata/pandas/issues/3449
+.. _GH3495: https://github.com/pydata/pandas/issues/3495
 .. _GH3493: https://github.com/pydata/pandas/issues/3493
 
 

pandas/core/format.py

@@ -820,21 +820,7 @@ def __init__(self, obj, path_or_buf, sep=",", na_rep='', float_format=None,
         self.blocks = self.obj._data.blocks
         ncols = sum(len(b.items) for b in self.blocks)
         self.data =[None] * ncols
-
-        if self.obj.columns.is_unique:
-            self.colname_map = dict((k,i) for i,k in  enumerate(self.obj.columns))
-        else:
-            ks = [set(x.items) for x in self.blocks]
-            u = len(reduce(lambda a,x: a.union(x),ks,set()))
-            t = sum(map(len,ks))
-            if u != t:
-                if len(set(self.cols)) != len(self.cols):
-                    raise NotImplementedError("duplicate columns with differing dtypes are unsupported")
-            else:
-                # if columns are not unique and we acces this,
-                # we're doing it wrong
-                pass
-
+        self.column_map = self.obj._data.get_items_map()
 
         if chunksize is None:
             chunksize = (100000/ (len(self.cols) or 1)) or 1
@@ -1034,18 +1020,13 @@ def _save_chunk(self, start_i, end_i):
 
         # create the data for a chunk
         slicer = slice(start_i,end_i)
-        if self.obj.columns.is_unique:
-            for i in range(len(self.blocks)):
-                b = self.blocks[i]
-                d = b.to_native_types(slicer=slicer, na_rep=self.na_rep, float_format=self.float_format)
-                for j, k in enumerate(b.items):
-                    # self.data is a preallocated list
-                    self.data[self.colname_map[k]] = d[j]
-        else:
-            # self.obj should contain a proper view of the dataframes
-            # with the specified ordering of cols if cols was specified
-            for i in range(len(self.obj.columns)):
-                self.data[i] = self.obj.icol(i).values[slicer].tolist()
+        for i in range(len(self.blocks)):
+            b = self.blocks[i]
+            d = b.to_native_types(slicer=slicer, na_rep=self.na_rep, float_format=self.float_format)
+            for i, item in enumerate(b.items):
+
+                # self.data is a preallocated list
+                self.data[self.column_map[b][i]] = d[i]
 
         ix = data_index.to_native_types(slicer=slicer, na_rep=self.na_rep, float_format=self.float_format)