ENH: HDFStore will retain index attributes (freq,tz,name) on recreation (GH3499_)

TST: added legacy_table_0.11 table and tests

Author: jreback

RELEASE.rst

@@ -38,6 +38,7 @@ pandas 0.11.1
   - Fixed various issues with internal pprinting code, the repr() for various objects
     including TimeStamp and *Index now produces valid python code strings and
     can be used to recreate the object, (GH3038_), (GH3379_), (GH3251_)
+  - ``HDFStore`` will retain index attributes (freq,tz,name) on recreation (GH3499_)
 
 **API Changes**
 

pandas/core/index.py

@@ -83,12 +83,12 @@ class Index(np.ndarray):
 
     _engine_type = _index.ObjectEngine
 
-    def __new__(cls, data, dtype=None, copy=False, name=None):
+    def __new__(cls, data, dtype=None, copy=False, name=None, **kwargs):
         from pandas.tseries.period import PeriodIndex
         if isinstance(data, np.ndarray):
             if issubclass(data.dtype.type, np.datetime64):
                 from pandas.tseries.index import DatetimeIndex
-                result = DatetimeIndex(data, copy=copy, name=name)
+                result = DatetimeIndex(data, copy=copy, name=name, **kwargs)
                 if dtype is not None and _o_dtype == dtype:
                     return Index(result.to_pydatetime(), dtype=_o_dtype)
                 else:
@@ -102,7 +102,7 @@ def __new__(cls, data, dtype=None, copy=False, name=None):
                 except TypeError:
                     pass
             elif isinstance(data, PeriodIndex):
-                return PeriodIndex(data, copy=copy, name=name)
+                return PeriodIndex(data, copy=copy, name=name, **kwargs)
 
             if issubclass(data.dtype.type, np.integer):
                 return Int64Index(data, copy=copy, dtype=dtype, name=name)
@@ -123,10 +123,10 @@ def __new__(cls, data, dtype=None, copy=False, name=None):
                 if (inferred.startswith('datetime') or
                         tslib.is_timestamp_array(subarr)):
                     from pandas.tseries.index import DatetimeIndex
-                    return DatetimeIndex(subarr, copy=copy, name=name)
+                    return DatetimeIndex(subarr, copy=copy, name=name, **kwargs)
 
                 elif inferred == 'period':
-                    return PeriodIndex(subarr, name=name)
+                    return PeriodIndex(subarr, name=name, **kwargs)
 
         subarr = subarr.view(cls)
         subarr.name = name

pandas/io/pytables.py

@@ -943,7 +943,8 @@ class IndexCol(object):
     is_searchable = False
 
     def __init__(self, values=None, kind=None, typ=None, cname=None, itemsize=None,
-                 name=None, axis=None, kind_attr=None, pos=None, **kwargs):
+                 name=None, axis=None, kind_attr=None, pos=None, freq=None, tz=None, 
+                 index_name=None, **kwargs):
         self.values = values
         self.kind = kind
         self.typ = typ
@@ -953,6 +954,9 @@ def __init__(self, values=None, kind=None, typ=None, cname=None, itemsize=None,
         self.kind_attr = kind_attr
         self.axis = axis
         self.pos = pos
+        self.freq = freq
+        self.tz = tz
+        self.index_name = None
         self.table = None
 
         if name is not None:
@@ -1023,7 +1027,22 @@ def convert(self, values, nan_rep):
             values = values[self.cname]
         except:
             pass
-        self.values = Index(_maybe_convert(values, self.kind))
+
+        kwargs = dict()
+        if self.freq is not None:
+            kwargs['freq'] = self.freq
+        if self.tz is not None:
+            kwargs['tz'] = self.tz
+        if self.name is not None:
+            kwargs['name'] = self.index_name
+        try:
+            self.values = Index(_maybe_convert(values, self.kind), **kwargs)
+        except:
+
+            # if the output freq is different that what we recorded, then infer it
+            if 'freq' in kwargs:
+                kwargs['freq'] = 'infer'
+            self.values = Index(_maybe_convert(values, self.kind), **kwargs)
         return self
 
     def take_data(self):
@@ -1098,6 +1117,30 @@ def validate_attr(self, append):
                 raise TypeError("incompatible kind in col [%s - %s]" %
                                 (existing_kind, self.kind))
 
+    def update_info(self, info):
+        """ set/update the info for this indexable with the key/value
+            if validate is True, then raise if an existing value does not match the value """
+
+        for key in ['freq','tz','name']:
+
+            value = getattr(self,key,None)
+
+            try:
+                idx = info[self.name]
+            except:
+                idx = info[self.name] = dict()
+        
+            existing_value = idx.get(key)
+            if key in idx and existing_value != value:
+                raise ValueError("invalid info for [%s] for [%s]"""
+                                 ", existing_value [%s] conflicts with new value [%s]" % (self.name,
+                                                                                          key,existing_value,value))
+
+            if value is not None or existing_value is not None:
+                idx[key] = value
+
+        return self
+
     def get_attr(self):
         """ set the kind for this colummn """
         self.kind = getattr(self.attrs, self.kind_attr, None)
@@ -2060,6 +2103,7 @@ def __init__(self, *args, **kwargs):
         self.non_index_axes = []
         self.values_axes    = []
         self.data_columns   = []
+        self.info           = dict()
         self.nan_rep        = None
         self.selection      = None
 
@@ -2173,18 +2217,20 @@ def values_cols(self):
 
     def set_attrs(self):
         """ set our table type & indexables """
-        self.attrs.table_type = self.table_type
-        self.attrs.index_cols = self.index_cols()
-        self.attrs.values_cols = self.values_cols()
+        self.attrs.table_type   = self.table_type
+        self.attrs.index_cols   = self.index_cols()
+        self.attrs.values_cols  = self.values_cols()
         self.attrs.non_index_axes = self.non_index_axes
         self.attrs.data_columns = self.data_columns
-        self.attrs.nan_rep = self.nan_rep
-        self.attrs.levels = self.levels
+        self.attrs.info         = self.info
+        self.attrs.nan_rep      = self.nan_rep
+        self.attrs.levels       = self.levels
 
     def get_attrs(self):
         """ retrieve our attributes """
         self.non_index_axes   = getattr(self.attrs,'non_index_axes',None) or []
         self.data_columns     = getattr(self.attrs,'data_columns',None)   or []
+        self.info             = getattr(self.attrs,'info',None)           or dict()
         self.nan_rep          = getattr(self.attrs,'nan_rep',None)
         self.levels           = getattr(self.attrs,'levels',None)         or []
         t = self.table
@@ -2221,8 +2267,17 @@ def indexables(self):
             d = self.description
             self._indexables = []
 
+            # info
+            info = getattr(self.attrs,'info',None) or dict()
+
             # index columns
-            self._indexables.extend([IndexCol(name=name, axis=axis, pos=i) for i, (axis, name) in enumerate(self.attrs.index_cols)])
+            def create_index(i, axis, name):
+                kwargs = dict( name=name, axis=axis, pos=i )
+                i = info.get(name)
+                if i is not None and len(i):
+                    kwargs.update(i)
+                return IndexCol(**kwargs)
+            self._indexables.extend([ create_index(i,axis,name) for i, (axis, name) in enumerate(self.attrs.index_cols)])
 
             # values columns
             dc = set(self.data_columns)
@@ -2379,7 +2434,8 @@ def create_axes(self, axes, obj, validate=True, nan_rep=None, data_columns=None,
             existing_table.infer_axes()
             axes         = [ a.axis for a in existing_table.index_axes]
             data_columns = existing_table.data_columns
-            nan_rep = existing_table.nan_rep
+            nan_rep      = existing_table.nan_rep
+            self.info    = existing_table.info
         else:
             existing_table = None
 
@@ -2421,7 +2477,7 @@ def create_axes(self, axes, obj, validate=True, nan_rep=None, data_columns=None,
                 self.non_index_axes.append((i, append_axis))
 
         # set axis positions (based on the axes)
-        self.index_axes = [index_axes_map[a].set_pos(j) for j,
+        self.index_axes = [index_axes_map[a].set_pos(j).update_info(self.info) for j,
                            a in enumerate(axes)]
         j = len(self.index_axes)
 
@@ -3042,10 +3098,10 @@ class AppendableNDimTable(AppendablePanelTable):
 def _convert_index(index):
     if isinstance(index, DatetimeIndex):
         converted = index.asi8
-        return IndexCol(converted, 'datetime64', _tables().Int64Col())
+        return IndexCol(converted, 'datetime64', _tables().Int64Col(), freq=getattr(index,'freq',None), tz=getattr(index,'tz',None))
     elif isinstance(index, (Int64Index, PeriodIndex)):
         atom = _tables().Int64Col()
-        return IndexCol(index.values, 'integer', atom)
+        return IndexCol(index.values, 'integer', atom, freq=getattr(index,'freq',None))
 
     if isinstance(index, MultiIndex):
         raise Exception('MultiIndex not supported here!')

pandas/io/tests/data/legacy_hdf/legacy_table_0.11.h5

@@ -7,6 +7,7 @@
 import datetime
 import numpy as np
 
+import pandas
 from pandas import (Series, DataFrame, Panel, MultiIndex, bdate_range,
                     date_range, Index)
 from pandas.io.pytables import HDFStore, get_store, Term, IncompatibilityWarning, PerformanceWarning
@@ -2041,6 +2042,22 @@ def test_select_iterator(self):
             result = concat(results)
             tm.assert_frame_equal(expected, result)
 
+    def test_retain_index_attributes(self):
+
+        # GH 3499, losing frequency info on index recreation
+        df = DataFrame(dict(A = Series(xrange(3), 
+                                       index=date_range('2000-1-1',periods=3,freq='H'))))
+
+        with ensure_clean(self.path) as store:
+            store.put('data', df, table=True)
+
+            result = store.get('data')
+            tm.assert_frame_equal(df,result)
+
+            for attr in ['freq','tz']:
+                for idx in ['index','columns']:
+                    self.assert_(getattr(getattr(df,idx),attr,None) == getattr(getattr(result,idx),attr,None))
+
     def test_panel_select(self):
 
         wp = tm.makePanel()
@@ -2437,6 +2454,16 @@ def test_legacy_0_10_read(self):
         finally:
             safe_close(store)
 
+    def test_legacy_0_11_read(self):
+        # legacy from 0.11
+        try:
+            store = HDFStore(tm.get_data_path('legacy_hdf/legacy_table_0.11.h5'), 'r')
+            df = store.select('df')
+            df1 = store.select('df1')
+            mi = store.select('mi')
+        finally:
+            safe_close(store)
+
     def test_copy(self):
 
         def do_copy(f = None, new_f = None, keys = None, propindexes = True, **kwargs):
@@ -2497,14 +2524,22 @@ def do_copy(f = None, new_f = None, keys = None, propindexes = True, **kwargs):
     def test_legacy_table_write(self):
         raise nose.SkipTest
 
-        # legacy table types
+        store = HDFStore(tm.get_data_path('legacy_hdf/legacy_table_%s.h5' % pandas.__version__), 'a')
+
         df = tm.makeDataFrame()
         wp = tm.makePanel()
 
-        store = HDFStore(tm.get_data_path('legacy_hdf/legacy_table.h5'), 'a')
+        index = MultiIndex(levels=[['foo', 'bar', 'baz', 'qux'],
+                                   ['one', 'two', 'three']],
+                           labels=[[0, 0, 0, 1, 1, 2, 2, 3, 3, 3],
+                                   [0, 1, 2, 0, 1, 1, 2, 0, 1, 2]],
+                           names=['foo', 'bar'])
+        df = DataFrame(np.random.randn(10, 3), index=index,
+                       columns=['A', 'B', 'C'])
+        store.append('mi', df)
 
-        self.assertRaises(Exception, store.append, 'df1', df)
-        self.assertRaises(Exception, store.append, 'wp1', wp)
+        df = DataFrame(dict(A = 'foo', B = 'bar'),index=range(10))
+        store.append('df', df, data_columns = ['B'], min_itemsize={'A' : 200 })
 
         store.close()