ENH: silently drop None in concat, memoize datetime integers in HDFStore conversion

Author: wesm

pandas/io/pytables.py

@@ -777,7 +777,14 @@ def _maybe_convert(values, val_kind):
 
 def _get_converter(kind):
     if kind == 'datetime':
-        return datetime.fromtimestamp
+        cache = {}
+        def convert_datetime(x):
+            if x in cache:
+                return cache[x]
+            else:
+                cache[x] = result = datetime.fromtimestamp(x)
+                return result
+        return convert_datetime
     else: # pragma: no cover
         raise ValueError('invalid kind %s' % kind)
 

pandas/tools/merge.py

@@ -610,7 +610,8 @@ def concat(objs, axis=0, join='outer', join_axes=None, ignore_index=False,
     objs : list or dict of Series, DataFrame, or Panel objects
         If a dict is passed, the sorted keys will be used as the `keys`
         argument, unless it is passed, in which case the values will be
-        selected (see below)
+        selected (see below). Any None objects will be dropped silently unless
+        they are all None in which case an Exception will be raised
     axis : {0, 1, ...}, default 0
         The axis to concatenate along
     join : {'inner', 'outer'}, default 'outer'
@@ -672,6 +673,12 @@ def __init__(self, objs, axis=0, join='outer', join_axes=None,
                 keys = sorted(objs)
             objs = [objs[k] for k in keys]
 
+        # filter Nones
+        objs = [obj for obj in objs if obj is not None]
+
+        if len(objs) == 0:
+            raise Exception('All objects passed were None')
+
         # consolidate data
         for obj in objs:
             if isinstance(obj, NDFrame):

pandas/tools/tests/test_merge.py

@@ -1096,6 +1096,13 @@ def test_concat_single_with_key(self):
         expected = concat([df, df], keys=['foo', 'bar'])
         tm.assert_frame_equal(result, expected[:10])
 
+    def test_concat_exclude_none(self):
+        df = DataFrame(np.random.randn(10, 4))
+
+        pieces = [df[:5], None, None, df[5:]]
+        result = concat(pieces)
+        tm.assert_frame_equal(result, df)
+        self.assertRaises(Exception, concat, [None, None])
 
 if __name__ == '__main__':
     import nose

scripts/hdfstore_panel_perf.py

@@ -1,11 +1,12 @@
 from pandas import *
 from pandas.util.testing import rands
 
-i, j, k = 1, 10000, 500
+i, j, k = 7, 771, 5532
 
 panel = Panel(np.random.randn(i, j, k),
               items=[rands(10) for _ in xrange(i)],
-              major_axis=[rands(10) for _ in xrange(j)],
+              major_axis=DateRange('1/1/2000', periods=j,
+                                   offset=datetools.Minute()),
               minor_axis=[rands(10) for _ in xrange(k)])