ENH: correctly interpret data column dtypes and raise NotImplementedError (in cases of unicode/datetime64/date)

Author: jreback

pandas/io/pytables.py

@@ -1100,9 +1100,19 @@ def set_kind(self):
     def set_atom(self, block, existing_col, min_itemsize, nan_rep, **kwargs):
         """ create and setup my atom from the block b """
 
-        self.values = list(block.items)
-        if block.dtype.name == 'object':
+        self.values   = list(block.items)
+        dtype         = block.dtype.name
+
+        if dtype == 'object':
+            inferred_type = lib.infer_dtype(block.values.flatten())
+            if inferred_type == 'unicode':
+                raise NotImplementedError("unicode is not implemented as a table column")
+            elif inferred_type == 'date':
+                raise NotImplementedError("date is not implemented as a table column")
+
             self.set_atom_object(block, existing_col, min_itemsize, nan_rep)
+        elif dtype == 'datetime64[ns]':
+            raise NotImplementedError("datetime64[ns] is not implemented as a table column")
         else:
             self.set_atom_data(block)
 
@@ -1531,6 +1541,12 @@ def create_axes(self, axes, obj, validate = True, nan_rep = None, columns = None
             nan_rep = 'nan'
         self.nan_rep = nan_rep
 
+        # convert the objects if we can to better divine dtypes
+        try:
+            obj = obj.convert_objects()
+        except:
+            pass
+
         # create axes to index and non_index
         index_axes_map = dict()
         for i, a in enumerate(obj.axes):
@@ -1608,6 +1624,8 @@ def create_axes(self, axes, obj, validate = True, nan_rep = None, columns = None
                 col.set_pos(j)
 
                 self.values_axes.append(col)
+            except (NotImplementedError):
+                raise
             except (Exception), detail:
                 raise Exception("cannot find the correct atom type -> [dtype->%s] %s" % (b.dtype.name,str(detail)))
             j += 1

pandas/io/tests/test_pytables.py

@@ -4,7 +4,7 @@
 import sys
 import warnings
 
-from datetime import datetime
+import datetime
 import numpy as np
 
 from pandas import (Series, DataFrame, Panel, MultiIndex, bdate_range,
@@ -13,7 +13,7 @@
 import pandas.util.testing as tm
 from pandas.tests.test_series import assert_series_equal
 from pandas.tests.test_frame import assert_frame_equal
-from pandas import concat, Timestamp
+from pandas import concat
 
 try:
     import tables
@@ -559,6 +559,42 @@ def test_big_table_frame(self):
 
         print "\nbig_table frame [%s] -> %5.2f" % (rows,time.time()-x)
 
+
+    def test_big_table2_frame(self):
+        # this is a really big table: 2.5m rows x 300 float columns, 20 string columns
+        raise nose.SkipTest('no big table2 frame')
+
+        # create and write a big table
+        print "\nbig_table2 start"
+        import time
+        start_time = time.time()
+        df = DataFrame(np.random.randn(2.5*1000*1000, 300), index = range(int(2.5*1000*1000)), columns = [ 'E%03d' % i for i in xrange(300) ])
+        for x in range(20):
+            df['String%03d' % x] = 'string%03d' % x
+
+        print "\nbig_table2 frame (creation of df) -> %5.2f" % (time.time()-start_time)
+        start_time = time.time()
+
+        from arb.common import profile
+        fn = 'big_table2.h5'
+
+        try:
+            
+            @profile.profile_func()
+            def f():
+                store = HDFStore(fn,mode = 'w')
+                store.append('df',df)
+                store.close()
+            
+            f()
+            rows = store.root.df.table.nrows
+            #recons = store.select('df')
+        finally:
+            pass
+            #os.remove(fn)
+
+        print "\nbig_table2 frame [%s] -> %5.2f" % (rows,time.time()-start_time)
+
     def test_big_table_panel(self):
         raise nose.SkipTest('no big table panel')
 
@@ -665,6 +701,15 @@ def _make_one_p4d():
         self.store.append('p4d_mixed', p4d)
         tm.assert_panel4d_equal(self.store.select('p4d_mixed'), p4d)
 
+    def test_unimplemented_dtypes_table_columns(self):
+        #### currently not supported dtypes ####
+        from pandas import Timestamp
+
+        for n,f in [ ('timestamp',Timestamp('20010102')), ('unicode',u'\u03c3'), ('datetime',datetime.datetime(2001,1,2)), ('date',datetime.date(2001,1,2)) ]:
+            df = tm.makeDataFrame()
+            df[n] = f
+            self.assertRaises(NotImplementedError, self.store.append, 'df1_%s' % n, df)
+
     def test_remove(self):
         ts = tm.makeTimeSeries()
         df = tm.makeDataFrame()
@@ -829,7 +874,7 @@ def test_terms(self):
             ('major_axis', '20121114'),
             ('major_axis', '>', '20121114'),
             (('major_axis', ['20121114','20121114']),),
-            ('major_axis', datetime(2012,11,14)),
+            ('major_axis', datetime.datetime(2012,11,14)),
             'major_axis>20121114',
             'major_axis>20121114',
             'major_axis>20121114',
@@ -936,14 +981,13 @@ def test_index_types(self):
         ser = Series(values, [0, 'y'])
         self._check_roundtrip(ser, func)
 
-        ser = Series(values, [datetime.today(), 0])
+        ser = Series(values, [datetime.datetime.today(), 0])
         self._check_roundtrip(ser, func)
 
         ser = Series(values, ['y', 0])
         self._check_roundtrip(ser, func)
 
-        from datetime import date
-        ser = Series(values, [date.today(), 'a'])
+        ser = Series(values, [datetime.date.today(), 'a'])
         self._check_roundtrip(ser, func)
 
         ser = Series(values, [1.23, 'b'])
@@ -955,7 +999,7 @@ def test_index_types(self):
         ser = Series(values, [1, 5])
         self._check_roundtrip(ser, func)
 
-        ser = Series(values, [datetime(2012, 1, 1), datetime(2012, 1, 2)])
+        ser = Series(values, [datetime.datetime(2012, 1, 1), datetime.datetime(2012, 1, 2)])
         self._check_roundtrip(ser, func)
 
     def test_timeseries_preepoch(self):
@@ -1352,7 +1396,7 @@ def test_legacy_table_write(self):
         store.close()
 
     def test_store_datetime_fractional_secs(self):
-        dt = datetime(2012, 1, 2, 3, 4, 5, 123456)
+        dt = datetime.datetime(2012, 1, 2, 3, 4, 5, 123456)
         series = Series([0], [dt])
         self.store['a'] = series
         self.assertEquals(self.store['a'].index[0], dt)