Merge pull request #4768 from jreback/hdf_dups

BUG: reading from a store with duplicate columns across dtypes would raise (GH4767)

Author: jreback

doc/source/release.rst

@@ -239,6 +239,7 @@ See :ref:`Internal Refactoring<whatsnew_0130.refactoring>`
     - ``read_hdf`` was not respecting as passed ``mode`` (:issue:`4504`)
     - appending a 0-len table will work correctly (:issue:`4273`)
     - ``to_hdf`` was raising when passing both arguments ``append`` and ``table`` (:issue:`4584`)
+    - reading from a store with duplicate columns across dtypes would raise (:issue:`4767`)
   - Fixed bug in tslib.tz_convert(vals, tz1, tz2): it could raise IndexError exception while
     trying to access trans[pos + 1] (:issue:`4496`)
   - The ``by`` argument now works correctly with the ``layout`` argument

pandas/io/pytables.py

@@ -3219,7 +3219,7 @@ def read(self, where=None, columns=None, **kwargs):
         if len(objs) == 1:
             wp = objs[0]
         else:
-            wp = concat(objs, axis=0, verify_integrity=True)
+            wp = concat(objs, axis=0, verify_integrity=False)
 
         # apply the selection filters & axis orderings
         wp = self.process_axes(wp, columns=columns)
@@ -3510,7 +3510,7 @@ def read(self, where=None, columns=None, **kwargs):
         if len(frames) == 1:
             df = frames[0]
         else:
-            df = concat(frames, axis=1, verify_integrity=True)
+            df = concat(frames, axis=1, verify_integrity=False)
 
         # apply the selection filters & axis orderings
         df = self.process_axes(df, columns=columns)

pandas/io/tests/test_pytables.py

@@ -2296,6 +2296,29 @@ def test_wide_table(self):
         wp = tm.makePanel()
         self._check_roundtrip_table(wp, assert_panel_equal)
 
+    def test_select_with_dups(self):
+
+
+        # single dtypes
+        df = DataFrame(np.random.randn(10,4),columns=['A','A','B','B'])
+        df.index = date_range('20130101 9:30',periods=10,freq='T')
+
+        with ensure_clean(self.path) as store:
+            store.append('df',df)
+            result = store.select('df')
+            assert_frame_equal(result,df)
+
+        # dups accross dtypes
+        df = concat([DataFrame(np.random.randn(10,4),columns=['A','A','B','B']),
+                     DataFrame(np.random.randint(0,10,size=20).reshape(10,2),columns=['A','C'])],
+                    axis=1)
+        df.index = date_range('20130101 9:30',periods=10,freq='T')
+
+        with ensure_clean(self.path) as store:
+            store.append('df',df)
+            result = store.select('df')
+            assert_frame_equal(result,df)
+
     def test_wide_table_dups(self):
         wp = tm.makePanel()
         with ensure_clean(self.path) as store: