BUG: fix major performance issue in DatetimeIndex.union affecting join performance on irregular indexes, remedying #1046

Author: wesm

pandas/core/generic.py

@@ -132,7 +132,8 @@ def groupby(self, by=None, axis=0, level=None, as_index=True, sort=True):
         return groupby(self, by, axis=axis, level=level, as_index=as_index,
                        sort=sort)
 
-    def convert(self, rule, method='pad', how='last', axis=0, as_index=True):
+    def convert(self, rule, method='pad', how='last', axis=0, as_index=True,
+                closed='right', label='right'):
         """
         Convenience method for frequency conversion and resampling of regular
         time-series data.
@@ -143,6 +144,10 @@ def convert(self, rule, method='pad', how='last', axis=0, as_index=True):
         how : string, method for down- or re-sampling, default 'last'
         method : string, method for upsampling, default 'pad'
         axis : int, optional, default 0
+        closed : {'right', 'left'}, default 'right'
+            Which side of bin interval is closed
+        label : {'right', 'left'}, default 'right'
+            Which bin edge label to label bucket with
         as_index : see synonymous argument of groupby
         """
         from pandas.core.groupby import TimeGrouper, translate_grouping
@@ -154,14 +159,11 @@ def convert(self, rule, method='pad', how='last', axis=0, as_index=True):
         if not isinstance(idx, DatetimeIndex):
             raise ValueError("Cannot call convert with non-DatetimeIndex")
 
-        if idx.offset is None:
-            raise ValueError("Cannot call convert with non-regular index")
-
         if not isinstance(rule, datetools.DateOffset):
             raise ValueError("Rule not a recognized offset")
 
-        interval = TimeGrouper(rule, label='right',
-                               closed='right', _obj=self)
+        interval = TimeGrouper(rule, label=label,
+                               closed=closed, _obj=self)
 
         currfreq = len(idx)
         targfreq = len(interval.binner) - 2 # since binner extends endpoints

pandas/core/index.py

@@ -457,8 +457,7 @@ def union(self, other):
 
             if len(indexer) > 0:
                 other_diff = other.values.take(indexer)
-                result = list(self) + list(other_diff)
-                # timsort wins
+                result = np.concatenate((self.values, other_diff))
                 try:
                     result.sort()
                 except Exception:

vb_suite/timeseries.py

@@ -2,13 +2,14 @@
 from datetime import datetime
 
 common_setup = """from pandas_vb_common import *
+N = 100000
+
 try:
-    rng = date_range('1/1/2000', periods=100000, freq='min')
+    rng = date_range('1/1/2000', periods=N, freq='min')
 except NameError:
-    rng = DateRange('1/1/2000', periods=100000,
-                    offset=datetools.Minute())
+    rng = DateRange('1/1/2000', periods=N, offset=datetools.Minute())
 
-ts = Series(np.random.randn(100000), index=rng)
+ts = Series(np.random.randn(N), index=rng)
 """
 
 #----------------------------------------------------------------------
@@ -28,3 +29,15 @@
 
 timeseries_1min_5min_mean = Benchmark("ts[:10000].convert('5min', how='mean')",
                                       common_setup)
+
+#----------------------------------------------------------------------
+# Irregular alignment
+
+setup = common_setup + """
+lindex = np.random.permutation(N)[:N // 2]
+rindex = np.random.permutation(N)[:N // 2]
+left = Series(ts.values.take(lindex), index=ts.index.take(lindex))
+right = Series(ts.values.take(rindex), index=ts.index.take(rindex))
+"""
+
+timeseries_add_irregular = Benchmark('left + right', setup)