Merge pull request #10527 from kawochen/BUG-FIX-9618

BUG: GH9618 in read_msgpack where DataFrame has duplicate column names

Author: jreback

doc/source/whatsnew/v0.17.0.txt

@@ -385,4 +385,6 @@ Bug Fixes
 
 - Bug in operator equal on Index not being consistent with Series (:issue:`9947`)
 
-- Reading "famafrench" data via ``DataReader`` results in HTTP 404 error because of the website url is changed (:issue:`10591`).
+- Reading "famafrench" data via ``DataReader`` results in HTTP 404 error because of the website url is changed (:issue:`10591`).
+
+- Bug in `read_msgpack` where DataFrame to decode has duplicate column names (:issue:`9618`)

pandas/io/packers.py

@@ -357,6 +357,7 @@ def encode(obj):
                     'klass': obj.__class__.__name__,
                     'axes': data.axes,
                     'blocks': [{'items': data.items.take(b.mgr_locs),
+                                'locs': b.mgr_locs.as_array,
                                 'values': convert(b.values),
                                 'shape': b.values.shape,
                                 'dtype': b.dtype.num,
@@ -485,9 +486,15 @@ def decode(obj):
         def create_block(b):
             values = unconvert(b['values'], dtype_for(b['dtype']),
                                b['compress']).reshape(b['shape'])
+
+            # locs handles duplicate column names, and should be used instead of items; see GH 9618
+            if 'locs' in b:
+                placement = b['locs']
+            else:
+                placement = axes[0].get_indexer(b['items'])
             return make_block(values=values,
                               klass=getattr(internals, b['klass']),
-                              placement=axes[0].get_indexer(b['items']))
+                              placement=placement)
 
         blocks = [create_block(b) for b in obj['blocks']]
         return globals()[obj['klass']](BlockManager(blocks, axes))