BUG: don't assume that each object contains every unique block type in concat, GH #708

Author: wesm

pandas/tools/merge.py

@@ -730,8 +730,7 @@ def _get_concatenated_data(self):
 
             new_blocks = []
             for kind in kinds:
-                klass_blocks = [mapping.get(kind) for mapping in blockmaps
-                                if kind in mapping]
+                klass_blocks = [mapping.get(kind) for mapping in blockmaps]
                 stacked_block = self._concat_blocks(klass_blocks)
                 new_blocks.append(stacked_block)
             new_data = BlockManager(new_blocks, self.new_axes)
@@ -766,7 +765,8 @@ def _get_reindexed_data(self):
         return reindexed_data
 
     def _concat_blocks(self, blocks):
-        concat_values = np.concatenate([b.values for b in blocks],
+        concat_values = np.concatenate([b.values for b in blocks
+                                        if b is not None],
                                        axis=self.axis)
 
         if self.axis > 0:
@@ -776,12 +776,13 @@ def _concat_blocks(self, blocks):
                                 'DataFrames')
             return make_block(concat_values, blocks[0].items, self.new_axes[0])
         else:
-            all_items = [b.items for b in blocks]
+            all_items = [b.items for b in blocks if b is not None]
             if self.axis == 0 and self.keys is not None:
                 offsets = np.r_[0, np.cumsum([len(x._data.axes[self.axis]) for
                                               x in self.objs])]
                 indexer = np.concatenate([offsets[i] + b.ref_locs
-                                          for i, b in enumerate(blocks)])
+                                          for i, b in enumerate(blocks)
+                                          if b is not None])
                 concat_items = self.new_axes[0].take(indexer)
             else:
                 concat_items = _concat_indexes(all_items)

pandas/tools/tests/test_merge.py

@@ -854,6 +854,17 @@ def test_concat_keys_specific_levels(self):
         self.assert_(np.array_equal(result.columns.levels[0], level))
         self.assertEqual(result.columns.names[0], 'group_key')
 
+    def test_concat_dataframe_keys_bug(self):
+        t1 = DataFrame({'value': Series([1,2,3],
+                       index=Index(['a', 'b', 'c'], name='id'))})
+        t2 = DataFrame({'value': Series([7, 8],
+                       index=Index(['a', 'b'], name = 'id'))})
+
+        # it works
+        result = concat([t1, t2], axis=1, keys=['t1', 't2'])
+        self.assertEqual(list(result.columns), [('t1', 'value'),
+                                                ('t2', 'value')])
+
     def test_concat_dict(self):
         frames = {'foo' : DataFrame(np.random.randn(4, 3)),
                   'bar' : DataFrame(np.random.randn(4, 3)),