support hierarchial path-name like keys in stores (e.g. foo/bar/bah)

support removal of sub-stores by a higer level key
	(e.g. store.remove('df') will remove everything in the '/df' namespace and below)
access to namespaces is ABSOLUTE (and a '/' will be assumed if not provided), e.g. store.get('df') will get '/df')
  thus is backwards compatible
slight change to the __repr__ to show the paths

Author: jreback

RELEASE.rst

@@ -68,6 +68,7 @@ pandas 0.10.0
   - Add ``normalize`` option to Series/DataFrame.asfreq (#2137)
   - SparseSeries and SparseDataFrame construction from empty and scalar
     values now no longer create dense ndarrays unnecessarily (#2322)
+  - ``HDFStore`` now supports hierarchial keys (#2397)
   - Support multiple query selection formats for ``HDFStore tables`` (#1996)
   - Support ``del store['df']`` syntax to delete HDFStores
   - Add multi-dtype support for ``HDFStore tables``

doc/source/io.rst

@@ -865,6 +865,22 @@ after data is already in the table (this may become automatic in the future or a
    store.create_table_index('df')
    store.handle.root.df.table
 
+Hierarchical Keys
+~~~~~~~~~~~~~~~~~
+
+Keys to a store can be specified as a string. These can be in a hierarchical path-name like format (e.g. ``foo/bar/bah``), which will generate a hierarchy of sub-stores (or ``Groups`` in PyTables parlance). Keys can be specified with out the leading '/' and are ALWAYS absolute (e.g. 'foo' refers to '/foo'). Removal operations can remove everying in the sub-store and BELOW, so be *careful*.
+
+.. ipython:: python
+
+   store.put('foo/bar/bah', df)
+   store.append('food/orange', df)
+   store.append('food/apple',  df)
+   store
+
+   # remove all nodes under this level
+   store.remove('food')
+   store	
+
 Storing Mixed Types in a Table
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

doc/source/v0.10.0.txt

@@ -63,6 +63,19 @@ Updated PyTables Support
 
   **Enhancements**
 
+       - added ability to hierarchical keys
+
+          .. ipython:: python
+
+            store.put('foo/bar/bah', df)
+            store.append('food/orange', df)
+            store.append('food/apple',  df)
+            store
+
+            # remove all nodes under this level
+            store.remove('food')
+	    store	
+
        - added mixed-dtype support!
 
           .. ipython:: python

pandas/io/pytables.py

@@ -90,7 +90,6 @@ def _tables():
 
     return _table_mod
 
-
 @contextmanager
 def get_store(path, mode='a', complevel=None, complib=None,
               fletcher32=False):
@@ -197,6 +196,11 @@ def __init__(self, path, mode='a', complevel=None, complib=None,
         self.filters = None
         self.open(mode=mode, warn=False)
 
+    @property
+    def root(self):
+        """ return the root node """
+        return self.handle.root
+
     def __getitem__(self, key):
         return self.get(key)
 
@@ -207,26 +211,32 @@ def __delitem__(self, key):
         return self.remove(key)
 
     def __contains__(self, key):
-        return hasattr(self.handle.root, key)
+        return hasattr(self.root, key)
 
     def __len__(self):
-        return len(self.handle.root._v_children)
+        return len(self.groups())
 
     def __repr__(self):
         output = '%s\nFile path: %s\n' % (type(self), self.path)
 
-        if len(self) > 0:
-            keys = []
+        groups = self.groups()
+        if len(groups) > 0:
+            keys   = []
             values = []
-            for k, v in sorted(self.handle.root._v_children.iteritems()):
-                kind = getattr(v._v_attrs,'pandas_type',None)
+            for n in sorted(groups, key = lambda x: x._v_name):
+                kind = getattr(n._v_attrs,'pandas_type',None)
 
-                keys.append(str(k))
+                keys.append(str(n._v_pathname))
 
+                # a group
                 if kind is None:
-                    values.append('unknown type')
+                    values.append('')
+
+                # a table
                 elif _is_table_type(v):
-                    values.append(str(create_table(self, v)))
+                    values.append(str(create_table(self, n)))
+                
+                # another type of pandas object
                 else:
                     values.append(_NAME_MAP[kind])
 
@@ -241,7 +251,7 @@ def keys(self):
         Return a (potentially unordered) list of the keys corresponding to the
         objects stored in the HDFStore
         """
-        return self.handle.root._v_children.keys()
+        return [ n._v_pathname[1:] for n in self.groups() ]
 
     def open(self, mode='a', warn=True):
         """
@@ -304,12 +314,10 @@ def get(self, key):
         -------
         obj : type of object stored in file
         """
-        exc_type = _tables().NoSuchNodeError
-        try:
-            group = getattr(self.handle.root, key)
-            return self._read_group(group)
-        except (exc_type, AttributeError):
+        group = self.get_node(key)
+        if group is None:
             raise KeyError('No object named %s in the file' % key)
+        return self._read_group(group)
 
     def select(self, key, where=None):
         """
@@ -322,11 +330,12 @@ def select(self, key, where=None):
         where : list of Term (or convertable) objects, optional
 
         """
-        group = getattr(self.handle.root, key, None)
+        group = self.get_node(key)
+        if group is None:
+            raise KeyError('No object named %s in the file' % key)
         if where is not None and not _is_table_type(group):
             raise Exception('can only select with where on objects written as tables')
-        if group is not None:
-            return self._read_group(group, where)
+        return self._read_group(group, where)
 
     def put(self, key, value, table=False, append=False,
             compression=None, **kwargs):
@@ -352,9 +361,6 @@ def put(self, key, value, table=False, append=False,
         self._write_to_group(key, value, table=table, append=append,
                              comp=compression, **kwargs)
 
-    def _get_handler(self, op, kind):
-        return getattr(self, '_%s_%s' % (op, kind))
-
     def remove(self, key, where=None):
         """
         Remove pandas object partially by specifying the where condition
@@ -372,15 +378,21 @@ def remove(self, key, where=None):
         number of rows removed (or None if not a Table)
 
         """
-        if where is None:
-            self.handle.removeNode(self.handle.root, key, recursive=True)
-        else:
-            group = getattr(self.handle.root, key, None)
-            if group is not None:
+        group = self.get_node(key)
+        if group is not None:
+
+            # remove the node
+            if where is None:
+                group = self.get_node(key)
+                group._f_remove(recursive=True)
+            
+            # delete from the table
+            else:
                 if not _is_table_type(group):
                     raise Exception('can only remove with where on objects written as tables')
                 t = create_table(self, group)
                 return t.delete(where)
+
         return None
 
     def append(self, key, value, **kwargs):
@@ -416,20 +428,52 @@ def create_table_index(self, key, **kwargs):
         if not _table_supports_index:
             raise("PyTables >= 2.3 is required for table indexing")
 
-        group = getattr(self.handle.root, key, None)
+        group = self.get_node(key)
         if group is None: return
 
         if not _is_table_type(group):
             raise Exception("cannot create table index on a non-table")
         create_table(self, group).create_index(**kwargs)
 
+    def groups(self):
+        """ return a list of all the groups (that are not themselves a pandas storage object) """
+        return [ g for g in self.handle.walkGroups() if getattr(g._v_attrs,'pandas_type',None) ]
+
+    def get_node(self, key):
+        """ return the node with the key or None if it does not exist """
+        try:
+            if not key.startswith('/'):
+                key = '/' + key
+            return self.handle.getNode(self.root,key)
+        except:
+            return None
+
+    ###### private methods ######
+
+    def _get_handler(self, op, kind):
+        return getattr(self, '_%s_%s' % (op, kind))
+
     def _write_to_group(self, key, value, table=False, append=False,
                         comp=None, **kwargs):
-        root = self.handle.root
-        if key not in root._v_children:
-            group = self.handle.createGroup(root, key)
-        else:
-            group = getattr(root, key)
+        group = self.get_node(key)
+        if group is None:
+            paths = key.split('/')
+
+            # recursively create the groups
+            path = '/'
+            if len(paths) > 1:
+                for p in paths[:-1]:
+                    new_path = path
+                    if not path.endswith('/'):
+                        new_path += '/'
+                    new_path += p
+                    group = self.get_node(new_path)
+                    if group is None:
+                        group = self.handle.createGroup(path, p)
+                    path  = new_path
+
+            # create the required group
+            group = self.handle.createGroup(path, paths[-1])
 
         kind = _TYPE_MAP[type(value)]
         if table or (append and _is_table_type(group)):

pandas/io/tests/test_pytables.py

@@ -58,25 +58,29 @@ def test_len_keys(self):
         self.store['b'] = tm.makeStringSeries()
         self.store['c'] = tm.makeDataFrame()
         self.store['d'] = tm.makePanel()
-        self.assertEquals(len(self.store), 4)
-        self.assert_(set(self.store.keys()) == set(['a', 'b', 'c', 'd']))
+        self.store['foo/bar'] = tm.makePanel()
+        self.assertEquals(len(self.store), 5)
+        self.assert_(set(self.store.keys()) == set(['a', 'b', 'c', 'd', 'foo/bar']))
 
     def test_repr(self):
         repr(self.store)
         self.store['a'] = tm.makeTimeSeries()
         self.store['b'] = tm.makeStringSeries()
         self.store['c'] = tm.makeDataFrame()
         self.store['d'] = tm.makePanel()
+        self.store['foo/bar'] = tm.makePanel()
         self.store.append('e', tm.makePanel())
         repr(self.store)
         str(self.store)
 
     def test_contains(self):
         self.store['a'] = tm.makeTimeSeries()
         self.store['b'] = tm.makeDataFrame()
+        self.store['foo/bar'] = tm.makeDataFrame()
         self.assert_('a' in self.store)
         self.assert_('b' in self.store)
         self.assert_('c' not in self.store)
+        self.assert_('foo/bar' in self.store)
 
     def test_reopen_handle(self):
         self.store['a'] = tm.makeTimeSeries()
@@ -94,13 +98,18 @@ def test_get(self):
         right = self.store['a']
         tm.assert_series_equal(left, right)
 
+        left = self.store.get('/a')
+        right = self.store['/a']
+        tm.assert_series_equal(left, right)
+
         self.assertRaises(KeyError, self.store.get, 'b')
 
     def test_put(self):
         ts = tm.makeTimeSeries()
         df = tm.makeTimeDataFrame()
         self.store['a'] = ts
         self.store['b'] = df[:10]
+        self.store['foo/bar/bah'] = df[:10]
         self.store.put('c', df[:10], table=True)
 
         # not OK, not a table
@@ -295,6 +304,19 @@ def test_remove(self):
         self.store.remove('b')
         self.assertEquals(len(self.store), 0)
 
+        # pathing
+        self.store['a'] = ts
+        self.store['b/foo'] = df
+        self.store.remove('foo')
+        self.store.remove('b/foo')
+        self.assertEquals(len(self.store), 1)
+
+        self.store['a'] = ts
+        self.store['b/foo'] = df
+        self.store.remove('b')
+        self.assertEquals(len(self.store), 1)
+
+
         # __delitem__
         self.store['a'] = ts
         self.store['b'] = df