Merge pull request pandas-dev#106 from manahl/fix_pandas_panel_storage

Fix pd.Panel storage.

Author: mildbyte

CHANGES.md

@@ -1,5 +1,9 @@
 ## Changelog
 
+### 1.21 (2016-03-08)
+
+  * Bugfix: #106 Fix Pandas Panel storage for panels with different dimensions
+
 ### 1.20 (2016-02-03)
 
   * Feature: #98 Add initial_image as optional parameter on tickstore write()

arctic/store/_pandas_ndarray_store.py

@@ -316,7 +316,7 @@ class PandasPanelStore(PandasDataFrameStore):
 
     def can_write(self, version, symbol, data):
         if isinstance(data, Panel):
-            frame = data.to_frame()
+            frame = data.to_frame(filter_observations=False)
             if np.any(frame.dtypes.values == 'object'):
                 return self.can_convert_to_records_without_objects(frame, symbol)
             return True
@@ -329,7 +329,7 @@ def write(self, arctic_lib, version, symbol, item, previous_version):
             raise ValueError('Cannot insert a zero size panel into mongo.')
         if not np.all(len(i.names) == 1 for i in item.axes):
             raise ValueError('Cannot insert panels with multiindexes')
-        item = item.to_frame()
+        item = item.to_frame(filter_observations=False)
         if len(set(item.dtypes)) == 1:
             # If all columns have the same dtype, we support non-string column names.
             # We know from above check that columns is not a multiindex.

tests/integration/store/test_pandas_store.py

@@ -538,6 +538,21 @@ def test_panel_save_read(library, df_size):
                 str(pn.axes[i].names) + "!=" + str(pn.axes[i].names)
 
 
+def test_panel_save_read_with_nans(library):
+    '''Ensure that nan rows are not dropped when calling to_frame.'''
+    df1 = DataFrame(data=np.arange(4).reshape((2, 2)), index=['r1', 'r2'], columns=['c1', 'c2'])
+    df2 = DataFrame(data=np.arange(6).reshape((3, 2)), index=['r1', 'r2', 'r3'], columns=['c1', 'c2'])
+    p_in = Panel(data=dict(i1=df1, i2=df2))
+
+    library.write('pandas', p_in)
+    p_out = library.read('pandas').data
+
+    assert p_in.shape == p_out.shape
+    # check_names is False because pandas helpfully names the axes for us.
+    assert_frame_equal(p_in.iloc[0], p_out.iloc[0], check_names=False)  
+    assert_frame_equal(p_in.iloc[1], p_out.iloc[1], check_names=False)  
+
+
 def test_save_read_ints(library):
     ts1 = DataFrame(index=[dt(2012, 1, 1) + dtd(hours=x) for x in range(5)],
                     data={'col1':np.arange(5), 'col2':np.arange(5)})

tests/unit/store/test_pandas_ndarray_store.py

@@ -63,7 +63,7 @@ def test_panel_converted_to_dataframe_and_stacked_to_write():
     with patch.object(PandasDataFrameStore, 'write') as mock_write:
         with patch('arctic.store._pandas_ndarray_store.DataFrame') as DF:
             store.write(sentinel.mlib, sentinel.version, sentinel.symbol, panel, sentinel.prev)
-    panel.to_frame.assert_called_with()
+    panel.to_frame.assert_called_with(filter_observations=False)
     DF.assert_called_with(panel.to_frame.return_value.stack.return_value)
     mock_write.assert_called_with(sentinel.mlib, sentinel.version, sentinel.symbol,
                                   DF.return_value, sentinel.prev)