ENH: support timezone data_columns in HDFStore (GH2852)

DOC: update release notes/whatsnew, added whatsnew 0.11.1 to index.rst

ENH: warn a FrequencyWarning if appending with a different frequency that existing

Author: jreback

RELEASE.rst

@@ -38,7 +38,12 @@ pandas 0.11.1
   - Fixed various issues with internal pprinting code, the repr() for various objects
     including TimeStamp and *Index now produces valid python code strings and
     can be used to recreate the object, (GH3038_), (GH3379_), (GH3251_)
-  - ``HDFStore`` will retain index attributes (freq,tz,name) on recreation (GH3499_)
+  - ``HDFStore``
+
+    - will retain index attributes (freq,tz,name) on recreation (GH3499_)
+    - will warn with a FrequencyWarning if you are attempting to append
+      an index with a different frequency than the existing
+    - support datelike columns with a timezone as data_columns (GH2852_)
 
 **API Changes**
 
@@ -88,6 +93,7 @@ pandas 0.11.1
 .. _GH3251: https://github.com/pydata/pandas/issues/3251
 .. _GH3379: https://github.com/pydata/pandas/issues/3379
 .. _GH3480: https://github.com/pydata/pandas/issues/3480
+.. _GH2852: https://github.com/pydata/pandas/issues/2852
 .. _GH3454: https://github.com/pydata/pandas/issues/3454
 .. _GH3457: https://github.com/pydata/pandas/issues/3457
 .. _GH3491: https://github.com/pydata/pandas/issues/3491
@@ -103,7 +109,7 @@ pandas 0.11.1
 .. _GH3461: https://github.com/pydata/pandas/issues/3461
 .. _GH3468: https://github.com/pydata/pandas/issues/3468
 .. _GH3448: https://github.com/pydata/pandas/issues/3448
-.. _GH3449: https://github.com/pydata/pandas/issues/3449
+.. _GH3499: https://github.com/pydata/pandas/issues/3499
 .. _GH3495: https://github.com/pydata/pandas/issues/3495
 .. _GH3492: https://github.com/pydata/pandas/issues/3492
 .. _GH3493: https://github.com/pydata/pandas/issues/3493

doc/source/v0.11.1.txt

@@ -1,6 +1,6 @@
-.. _whatsnew_0120:
+.. _whatsnew_0111:
 
-v0.12.0 (??)
+v0.11.1 (??)
 ------------------------
 
 This is a major release from 0.11.0 and includes many new features and
@@ -12,13 +12,21 @@ API changes
 
 Enhancements
 ~~~~~~~~~~~~
-  - pd.read_html() can now parse HTML string, files or urls and return dataframes
+  - ``pd.read_html()`` can now parse HTML string, files or urls and return dataframes
     courtesy of @cpcloud. (GH3477_)
+  - ``HDFStore``
+
+    - will retain index attributes (freq,tz,name) on recreation (GH3499_)
+    - will warn with a FrequencyWarning if you are attempting to append
+      an index with a different frequency than the existing
+    - support datelike columns with a timezone as data_columns (GH2852_)
 
 See the `full release notes
 <https://github.com/pydata/pandas/blob/master/RELEASE.rst>`__ or issue tracker
 on GitHub for a complete list.
 
 .. _GH2437: https://github.com/pydata/pandas/issues/2437
+.. _GH2852: https://github.com/pydata/pandas/issues/2852
 .. _GH3477: https://github.com/pydata/pandas/issues/3477
 .. _GH3492: https://github.com/pydata/pandas/issues/3492
+.. _GH3499: https://github.com/pydata/pandas/issues/3499

doc/source/whatsnew.rst

@@ -18,6 +18,8 @@ These are new features and improvements of note in each release.
 
 .. include:: v0.12.0.txt
 
+.. include:: v0.11.1.txt
+
 .. include:: v0.11.0.txt
 
 .. include:: v0.10.1.txt

pandas/io/pytables.py

@@ -42,6 +42,11 @@ class IncompatibilityWarning(Warning): pass
 where criteria is being ignored as this version [%s] is too old (or not-defined),
 read the file in and write it out to a new file to upgrade (with the copy_to method)
 """
+class FrequencyWarning(Warning): pass
+frequency_doc = """
+the frequency of the existing index is [%s] which conflicts with the new freq [%s],
+resetting the frequency to None
+"""
 class PerformanceWarning(Warning): pass
 performance_doc = """
 your performance may suffer as PyTables will pickle object types that it cannot map
@@ -149,9 +154,12 @@ def get_store(path, mode='a', complevel=None, complib=None,
 
 ### interface to/from ###
 
-def to_hdf(path_or_buf, key, value, mode=None, complevel=None, complib=None, **kwargs):
+def to_hdf(path_or_buf, key, value, mode=None, complevel=None, complib=None, append=None, **kwargs):
     """ store this object, close it if we opened it """
-    f = lambda store: store.put(key, value, **kwargs)
+    if append:
+        f = lambda store: store.append(key, value, **kwargs)
+    else:
+        f = lambda store: store.put(key, value, **kwargs)
 
     if isinstance(path_or_buf, basestring):
         with get_store(path_or_buf, mode=mode, complevel=complevel, complib=complib) as store:
@@ -941,6 +949,7 @@ class IndexCol(object):
     is_an_indexable = True
     is_data_indexable = True
     is_searchable = False
+    _info_fields = ['freq','tz','name']
 
     def __init__(self, values=None, kind=None, typ=None, cname=None, itemsize=None,
                  name=None, axis=None, kind_attr=None, pos=None, freq=None, tz=None, 
@@ -1121,7 +1130,7 @@ def update_info(self, info):
         """ set/update the info for this indexable with the key/value
             if validate is True, then raise if an existing value does not match the value """
 
-        for key in ['freq','tz','name']:
+        for key in self._info_fields:
 
             value = getattr(self,key,None)
 
@@ -1132,15 +1141,31 @@ def update_info(self, info):
 
             existing_value = idx.get(key)
             if key in idx and existing_value != value:
-                raise ValueError("invalid info for [%s] for [%s]"""
-                                 ", existing_value [%s] conflicts with new value [%s]" % (self.name,
-                                                                                          key,existing_value,value))
 
-            if value is not None or existing_value is not None:
-                idx[key] = value
+                # frequency just warn
+                if key == 'freq':
+                    ws = frequency_doc % (existing_value,value)
+                    warnings.warn(ws, FrequencyWarning)
+
+                    # reset
+                    idx[key] = None
+
+                else:
+                    raise ValueError("invalid info for [%s] for [%s]"""
+                                     ", existing_value [%s] conflicts with new value [%s]" % (self.name,
+                                                                                              key,existing_value,value))
+            else:
+                if value is not None or existing_value is not None:
+                    idx[key] = value
 
         return self
 
+    def set_info(self, info):
+        """ set my state from the passed info """
+        idx = info.get(self.name)
+        if idx is not None:
+            self.__dict__.update(idx)
+
     def get_attr(self):
         """ set the kind for this colummn """
         self.kind = getattr(self.attrs, self.kind_attr, None)
@@ -1180,6 +1205,7 @@ class DataCol(IndexCol):
     is_an_indexable = False
     is_data_indexable = False
     is_searchable = False
+    _info_fields = ['tz']
 
     @classmethod
     def create_for_block(cls, i=None, name=None, cname=None, version=None, **kwargs):
@@ -1249,7 +1275,7 @@ def set_kind(self):
             if self.typ is None:
                 self.typ = getattr(self.description,self.cname,None)
 
-    def set_atom(self, block, existing_col, min_itemsize, nan_rep, **kwargs):
+    def set_atom(self, block, existing_col, min_itemsize, nan_rep, info, **kwargs):
         """ create and setup my atom from the block b """
 
         self.values = list(block.items)
@@ -1264,10 +1290,27 @@ def set_atom(self, block, existing_col, min_itemsize, nan_rep, **kwargs):
                 "[date] is not implemented as a table column")
         elif inferred_type == 'datetime':
             if getattr(rvalues[0],'tzinfo',None) is not None:
+
+                # if this block has more than one timezone, raise
+                if len(set([r.tzinfo for r in rvalues])) != 1:
+                    raise TypeError(
+                        "too many timezones in this block, create separate data columns")
+
+                # convert this column to datetime64[ns] utc, and save the tz
+                index = DatetimeIndex(rvalues)
+                tz = getattr(index,'tz',None)
+                if tz is None:
+                    raise TypeError(
+                        "invalid timezone specification")
+
+                values = index.tz_convert('UTC').values.view('i8')
+                self.tz = tz
+                self.update_info(info)
+                self.set_atom_datetime64(block, values.reshape(block.values.shape))
+
+            else:
                 raise TypeError(
-                    "timezone support on datetimes is not yet implemented as a table column")
-            raise TypeError(
-                "[datetime] is not implemented as a table column")
+                    "[datetime] is not implemented as a table column")
         elif inferred_type == 'unicode':
             raise TypeError(
                 "[unicode] is not implemented as a table column")
@@ -1347,10 +1390,12 @@ def set_atom_data(self, block):
     def get_atom_datetime64(self, block):
         return _tables().Int64Col(shape=block.shape[0])
 
-    def set_atom_datetime64(self, block):
+    def set_atom_datetime64(self, block, values = None):
         self.kind = 'datetime64'
         self.typ = self.get_atom_datetime64(block)
-        self.set_data(block.values.view('i8'), 'datetime64')
+        if values is None:
+            values = block.values.view('i8')
+        self.set_data(values, 'datetime64')
 
     @property
     def shape(self):
@@ -1389,7 +1434,18 @@ def convert(self, values, nan_rep):
 
             # reverse converts
             if self.dtype == 'datetime64':
-                self.data = np.asarray(self.data, dtype='M8[ns]')
+                # recreate the timezone
+                if self.tz is not None:
+
+                    # data should be 2-dim here
+                    # we stored as utc, so just set the tz
+
+                    index = DatetimeIndex(self.data.ravel(),tz='UTC').tz_convert(self.tz)
+                    self.data = np.array(index.tolist(),dtype=object).reshape(self.data.shape)
+
+                else:
+                    self.data = np.asarray(self.data, dtype='M8[ns]')
+
             elif self.dtype == 'date':
                 self.data = np.array(
                     [date.fromtimestamp(v) for v in self.data], dtype=object)
@@ -2267,17 +2323,8 @@ def indexables(self):
             d = self.description
             self._indexables = []
 
-            # info
-            info = getattr(self.attrs,'info',None) or dict()
-
             # index columns
-            def create_index(i, axis, name):
-                kwargs = dict( name=name, axis=axis, pos=i )
-                i = info.get(name)
-                if i is not None and len(i):
-                    kwargs.update(i)
-                return IndexCol(**kwargs)
-            self._indexables.extend([ create_index(i,axis,name) for i, (axis, name) in enumerate(self.attrs.index_cols)])
+            self._indexables.extend([ IndexCol(name=name,axis=axis,pos=i) for i, (axis, name) in enumerate(self.attrs.index_cols)])
 
             # values columns
             dc = set(self.data_columns)
@@ -2370,6 +2417,7 @@ def read_axes(self, where, **kwargs):
 
         # convert the data
         for a in self.axes:
+            a.set_info(self.info)
             a.convert(values, nan_rep=self.nan_rep)
 
         return True
@@ -2535,6 +2583,7 @@ def create_axes(self, axes, obj, validate=True, nan_rep=None, data_columns=None,
                              existing_col=existing_col,
                              min_itemsize=min_itemsize,
                              nan_rep=nan_rep,
+                             info=self.info,
                              **kwargs)
                 col.set_pos(j)
 
@@ -2654,6 +2703,7 @@ def read_column(self, column, where = None, **kwargs):
 
                 # column must be an indexable or a data column
                 c = getattr(self.table.cols, column)
+                a.set_info(self.info)
                 return Series(a.convert(c[:], nan_rep=self.nan_rep).take_data())
 
         raise KeyError("column [%s] not found in the table" % column)
@@ -3365,6 +3415,8 @@ def convert_value(self, v):
 
         if self.kind == 'datetime64' or self.kind == 'datetime' :
             v = lib.Timestamp(v)
+            if v.tz is not None:
+                v = v.tz_convert('UTC')
             return [v.value, v]
         elif isinstance(v, datetime) or hasattr(v, 'timetuple') or self.kind == 'date':
             v = time.mktime(v.timetuple())

pandas/io/tests/test_pytables.py

@@ -10,7 +10,9 @@
 import pandas
 from pandas import (Series, DataFrame, Panel, MultiIndex, bdate_range,
                     date_range, Index)
-from pandas.io.pytables import HDFStore, get_store, Term, IncompatibilityWarning, PerformanceWarning
+from pandas.io.pytables import (HDFStore, get_store, Term, 
+                                IncompatibilityWarning, PerformanceWarning,
+                                FrequencyWarning)
 import pandas.util.testing as tm
 from pandas.tests.test_series import assert_series_equal
 from pandas.tests.test_frame import assert_frame_equal
@@ -1260,16 +1262,48 @@ def test_unimplemented_dtypes_table_columns(self):
             self.assertRaises(TypeError, store.append, 'df_unimplemented', df)
 
     def test_table_append_with_timezones(self):
-        # not implemented yet
 
         with ensure_clean(self.path) as store:
 
-            # check with mixed dtypes
-            df = DataFrame(dict(A = Timestamp('20130102',tz='US/Eastern')),index=range(5))
-
-            # timezones not yet supported
+            def compare(a,b):
+                tm.assert_frame_equal(a,b)
+                
+                # compare the zones on each element
+                for c in a.columns:
+                    for i in a.index:
+                        a_e = a[c][i]
+                        b_e = b[c][i]
+                        if not (a_e == b_e and a_e.tz == b_e.tz):
+                            raise AssertionError("invalid tz comparsion [%s] [%s]" % (a_e,b_e))
+
+            from datetime import timedelta
+
+            _maybe_remove(store, 'df_tz')
+            df = DataFrame(dict(A = [ Timestamp('20130102 2:00:00',tz='US/Eastern') + timedelta(hours=1)*i for i in range(5) ]))
+            store.append('df_tz',df,data_columns=['A'])
+            compare(store['df_tz'],df)
+
+            # select with tz aware
+            compare(store.select('df_tz',where=Term('A','>=',df.A[3])),df[df.A>=df.A[3]])
+
+            _maybe_remove(store, 'df_tz')
+            df = DataFrame(dict(A = Timestamp('20130102',tz='US/Eastern'), B = Timestamp('20130103',tz='US/Eastern')),index=range(5))
+            store.append('df_tz',df)
+            compare(store['df_tz'],df)
+
+            _maybe_remove(store, 'df_tz')
+            df = DataFrame(dict(A = Timestamp('20130102',tz='US/Eastern'), B = Timestamp('20130102',tz='EET')),index=range(5))
             self.assertRaises(TypeError, store.append, 'df_tz', df)
 
+            # this is ok
+            _maybe_remove(store, 'df_tz')
+            store.append('df_tz',df,data_columns=['A','B'])
+            compare(store['df_tz'],df)
+
+            # can't append with diff timezone
+            df = DataFrame(dict(A = Timestamp('20130102',tz='US/Eastern'), B = Timestamp('20130102',tz='CET')),index=range(5))
+            self.assertRaises(ValueError, store.append, 'df_tz', df)
+
     def test_remove(self):
 
         with ensure_clean(self.path) as store:
@@ -2049,6 +2083,7 @@ def test_retain_index_attributes(self):
                                        index=date_range('2000-1-1',periods=3,freq='H'))))
 
         with ensure_clean(self.path) as store:
+            _maybe_remove(store,'data')
             store.put('data', df, table=True)
 
             result = store.get('data')
@@ -2058,6 +2093,34 @@ def test_retain_index_attributes(self):
                 for idx in ['index','columns']:
                     self.assert_(getattr(getattr(df,idx),attr,None) == getattr(getattr(result,idx),attr,None))
 
+
+            # try to append a table with a different frequency
+            warnings.filterwarnings('ignore', category=FrequencyWarning)
+            df2 = DataFrame(dict(A = Series(xrange(3), 
+                                            index=date_range('2002-1-1',periods=3,freq='D'))))
+            store.append('data',df2)
+            warnings.filterwarnings('always', category=FrequencyWarning)
+
+            self.assert_(store.get_storer('data').info['index']['freq'] is None)
+
+            # this is ok
+            _maybe_remove(store,'df2')
+            df2 = DataFrame(dict(A = Series(xrange(3), 
+                                            index=[Timestamp('20010101'),Timestamp('20010102'),Timestamp('20020101')])))
+            store.append('df2',df2)
+            df3 = DataFrame(dict(A = Series(xrange(3),index=date_range('2002-1-1',periods=3,freq='D')))) 
+            store.append('df2',df3)
+
+    def test_retain_index_attributes2(self):
+
+        with tm.ensure_clean(self.path) as path:
+            warnings.filterwarnings('ignore', category=FrequencyWarning)
+            df  = DataFrame(dict(A = Series(xrange(3), index=date_range('2000-1-1',periods=3,freq='H'))))
+            df.to_hdf(path,'data',mode='w',append=True)
+            df2 = DataFrame(dict(A = Series(xrange(3), index=date_range('2002-1-1',periods=3,freq='D'))))
+            df2.to_hdf(path,'data',append=True)
+            warnings.filterwarnings('always', category=FrequencyWarning)
+
     def test_panel_select(self):
 
         wp = tm.makePanel()