BUG: correctly store float32 dtypes (that are not-mixed with float64 dtypes)

Author: jreback

RELEASE.rst

@@ -68,6 +68,7 @@ pandas 0.10.1
       is ``Int64``), (closes GH512_)
     - handle Timestamp correctly in data_columns (closes GH2637_)
     - contains correctly matches on non-natural names
+    - correctly store ``float32`` dtypes in tables (if not other float types in the same table)
   - Fix DataFrame.info bug with UTF8-encoded columns. (GH2576_)
   - Fix DatetimeIndex handling of FixedOffset tz (GH2604_)
   - More robust detection of being in IPython session for wide DataFrame

doc/source/v0.10.1.txt

@@ -126,6 +126,8 @@ Multi-table creation via ``append_to_multiple`` and selection via ``select_as_mu
 - You can pass ``expectedrows=an integer`` to the first ``append``, to set the TOTAL number of expectedrows that ``PyTables`` will expected. This will optimize read/write performance.
 - ``Select`` now supports passing ``start`` and ``stop`` to provide selection space limiting in selection.
 
+**Bug Fixes**
+- ``HDFStore`` tables can now store ``float32`` types correctly (cannot be mixed with ``float64`` however)
 
 See the `full release notes
 <https://github.com/pydata/pandas/blob/master/RELEASE.rst>`__ or issue tracker

pandas/io/pytables.py

@@ -1129,7 +1129,7 @@ def get_atom_data(self, block):
     def set_atom_data(self, block):
         self.kind = block.dtype.name
         self.typ = self.get_atom_data(block)
-        self.set_data(block.values.astype(self.typ._deftype))
+        self.set_data(block.values.astype(self.typ.type))
 
     def get_atom_datetime64(self, block):
         return _tables().Int64Col(shape=block.shape[0])
@@ -2116,6 +2116,22 @@ def get_object(self, obj):
         """ return the data for this obj """
         return obj
 
+    def convert_objects(self, obj):
+        """ attempt to convert any object fields; don't touch other fields
+            if we are converting anything, copy the object and modify the copy """
+        new_obj = None
+        convert_f = lambda x: lib.maybe_convert_objects(x, convert_datetime=True)
+
+        for col, s in obj.iteritems():
+            if s.dtype == np.object_:
+                if new_obj is None:
+                    new_obj = obj.copy()
+                new_obj[col] = convert_f(s)
+                    
+        if new_obj is not None:
+            return new_obj
+        return obj
+
     def create_axes(self, axes, obj, validate=True, nan_rep=None, data_columns=None, min_itemsize=None, **kwargs):
         """ create and return the axes
               leagcy tables create an indexable column, indexable index, non-indexable fields
@@ -2162,10 +2178,7 @@ def create_axes(self, axes, obj, validate=True, nan_rep=None, data_columns=None,
         self.nan_rep = nan_rep
 
         # convert the objects if we can to better divine dtypes
-        try:
-            obj = obj.convert_objects()
-        except:
-            pass
+        obj = self.convert_objects(obj)
 
         # create axes to index and non_index
         index_axes_map = dict()
@@ -2767,6 +2780,9 @@ def get_object(self, obj):
             obj = obj.transpose(*self.data_orientation)
         return obj
 
+    def convert_objects(self, obj):
+        return obj
+
     @property
     def is_transposed(self):
         return self.data_orientation != tuple(range(self.ndim))

pandas/io/tests/test_pytables.py

@@ -886,13 +886,25 @@ def test_table_values_dtypes_roundtrip(self):
         # incompatible dtype
         self.assertRaises(Exception, self.store.append, 'df_i8', df1)
 
-        #df1 = DataFrame({'a': Series([1, 2, 3], dtype='f4')})
-        #self.store.append('df_f4', df1)
-        #assert df1.dtypes == self.store['df_f4'].dtypes
-
-        #df2 = DataFrame({'a': Series([1, 2, 3], dtype='i4')})
-        #self.store.append('df_i4', df2)
-        #assert df2.dtypes == self.store['df_i4'].dtypes
+        # check creation/storage/retrieval of float32 (a bit hacky to actually create them thought)
+        df1 = DataFrame(np.array([[1],[2],[3]],dtype='f4'),columns = ['A'])
+        self.store.append('df_f4', df1)
+        assert df1.dtypes == self.store['df_f4'].dtypes
+        assert df1.dtypes[0] == 'float32'
+
+        # check with mixed dtypes (but not multi float types)
+        df1 = DataFrame(np.array([[1],[2],[3]],dtype='f4'),columns = ['float32'])
+        df1['string'] = 'foo'
+        self.store.append('df_mixed_dtypes1', df1)
+        assert (df1.dtypes == self.store['df_mixed_dtypes1'].dtypes).all() == True
+        assert df1.dtypes[0] == 'float32'
+        assert df1.dtypes[1] == 'object'
+
+        ### this is not supported, e.g. mixed float32/float64 blocks ###
+        #df1 = DataFrame(np.array([[1],[2],[3]],dtype='f4'),columns = ['float32'])
+        #df1['float64'] = 1.0
+        #self.store.append('df_mixed_dtypes2', df1)
+        #assert df1.dtypes == self.store['df_mixed_dtypes2'].dtypes).all() == True
 
     def test_table_mixed_dtypes(self):