Saving datetime64 with timezone to HDF5

[witosx]

An attempt to store a dataframe with datetime64+tz column (introduced in pandas 0.17) results in a TypeError exception

import pandas as pd

df = pd.DataFrame(pd.date_range('2015-01-01', '2015-01-02', freq='T'), columns=['time'])
df.time = df.time.dt.tz_localize('UTC')
df.dtypes

time    datetime64[ns, UTC]
dtype: object

df.to_hdf('/tmp/data.h5', 'data')

---------------------------------------------------------------------------

TypeError                                 Traceback (most recent call last)

<ipython-input-3-871011f2131e> in <module>()
----> 1 df.to_hdf('/tmp/data.h5', 'data')


/usr/lib/python3.5/site-packages/pandas/core/generic.py in to_hdf(self, path_or_buf, key, **kwargs)
    936 
    937         from pandas.io import pytables
--> 938         return pytables.to_hdf(path_or_buf, key, self, **kwargs)
    939 
    940     def to_msgpack(self, path_or_buf=None, **kwargs):


/usr/lib/python3.5/site-packages/pandas/io/pytables.py in to_hdf(path_or_buf, key, value, mode, complevel, complib, append, **kwargs)
    268         with HDFStore(path_or_buf, mode=mode, complevel=complevel,
    269                        complib=complib) as store:
--> 270             f(store)
    271     else:
    272         f(path_or_buf)


/usr/lib/python3.5/site-packages/pandas/io/pytables.py in <lambda>(store)
    263         f = lambda store: store.append(key, value, **kwargs)
    264     else:
--> 265         f = lambda store: store.put(key, value, **kwargs)
    266 
    267     if isinstance(path_or_buf, string_types):


/usr/lib/python3.5/site-packages/pandas/io/pytables.py in put(self, key, value, format, append, **kwargs)
    825             format = get_option("io.hdf.default_format") or 'fixed'
    826         kwargs = self._validate_format(format, kwargs)
--> 827         self._write_to_group(key, value, append=append, **kwargs)
    828 
    829     def remove(self, key, where=None, start=None, stop=None):


/usr/lib/python3.5/site-packages/pandas/io/pytables.py in _write_to_group(self, key, value, format, index, append, complib, encoding, **kwargs)
   1263 
   1264         # write the object
-> 1265         s.write(obj=value, append=append, complib=complib, **kwargs)
   1266 
   1267         if s.is_table and index:


/usr/lib/python3.5/site-packages/pandas/io/pytables.py in write(self, obj, **kwargs)
   2803             # I have no idea why, but writing values before items fixed #2299
   2804             blk_items = data.items.take(blk.mgr_locs)
-> 2805             self.write_array('block%d_values' % i, blk.values, items=blk_items)
   2806             self.write_index('block%d_items' % i, blk_items)
   2807 


/usr/lib/python3.5/site-packages/pandas/io/pytables.py in write_array(self, key, value, items)
   2584                         self.group, key)._v_attrs.value_type = 'timedelta64'
   2585                 else:
-> 2586                     self._handle.create_array(self.group, key, value)
   2587 
   2588         getattr(self.group, key)._v_attrs.transposed = transposed


/usr/lib/python3.5/site-packages/tables/file.py in create_array(self, where, name, obj, title, byteorder, createparents, atom, shape)
   1135                                     strides=(0,)*len(shape))
   1136         else:
-> 1137             flavor = flavor_of(obj)
   1138             # use a temporary object because converting obj at this stage
   1139             # breaks some test. This is soultion performs a double,


/usr/lib/python3.5/site-packages/tables/flavor.py in flavor_of(array)
    194     raise TypeError(
    195         "objects of type ``%s`` are not supported in this context, sorry; "
--> 196         "supported objects are: %s" % (type_name, supported_descs))
    197 
    198 


TypeError: objects of type ``DatetimeIndex`` are not supported in this context, sorry; supported objects are: NumPy array, record or scalar; homogeneous list or tuple, integer, float, complex or bytes

pd.show_versions()

INSTALLED VERSIONS
------------------
commit: None
python: 3.5.0.final.0
python-bits: 64
OS: Linux
OS-release: 4.2.2-1-ARCH
machine: x86_64
processor: 
byteorder: little
LC_ALL: None
LANG: pl_PL.utf8

pandas: 0.17.0
nose: 1.3.7
pip: 7.1.2
setuptools: 18.3.2
Cython: 0.23.3
numpy: 1.9.3
scipy: 0.16.0
statsmodels: 0.6.1
IPython: 4.0.0
sphinx: 1.3.1
patsy: 0.4.0
dateutil: 2.4.2
pytz: 2015.6
blosc: None
bottleneck: 1.0.0
tables: 3.2.2
numexpr: 2.4.4
matplotlib: 1.4.3
openpyxl: 2.2.6
xlrd: 0.9.4
xlwt: 1.0.0
xlsxwriter: 0.7.5
lxml: 3.4.4
bs4: 4.4.0
html5lib: 0.9999999
httplib2: None
apiclient: None
sqlalchemy: 1.0.8
pymysql: None
psycopg2: 2.6.1 (dt dec pq3 ext lo64)

[jreback]

This is not supported on fixed stores. I suppose it could be implemented, but is somewhat non-trivial. You should use format='table'

In [11]: df.to_hdf('/tmp/data.h5', 'data',format='table')

In [12]: pd.read_hdf('/tmp/data.h5','data').dtypes
Out[12]: 
time    datetime64[ns, UTC]
dtype: object

[jorisvandenbossche]

@jreback I think this error message is rather confusing. As you just have a column of datetime values (but timezone aware in this case), while the error message says that DatetimeIndex is not supported. And it is not that datetime values are not supported, it is only the timezone part of it.