ENH: don't populate hash table in index engine if > 1e6 elements, to save memory and speed. close #1160

Author: wesm

pandas/__init__.py

@@ -36,4 +36,5 @@
 
 from pandas.tools.merge import merge, concat
 from pandas.tools.pivot import pivot_table, crosstab
+from pandas.tools.plotting import scatter_matrix
 from pandas.tools.describe import value_range

pandas/src/engines.pyx

@@ -44,18 +44,27 @@ def get_value_at(ndarray arr, object loc):
 def set_value_at(ndarray arr, object loc, object val):
     return util.set_value_at(arr, loc, val)
 
+
+# Don't populate hash tables in monotonic indexes larger than this
+cdef int _SIZE_CUTOFF = 1000000
+
+
 cdef class IndexEngine:
 
     cdef readonly:
         object index_weakref
         HashTable mapping
+        bint over_size_threshold
 
     cdef:
         bint unique, monotonic
         bint initialized, monotonic_check, unique_check
 
     def __init__(self, index_weakref):
         self.index_weakref = index_weakref
+
+        self.over_size_threshold = len(index_weakref()) >= _SIZE_CUTOFF
+
         self.initialized = 0
         self.monotonic_check = 0
 
@@ -101,6 +110,15 @@ cdef class IndexEngine:
         if is_definitely_invalid_key(val):
             raise TypeError
 
+        if self.over_size_threshold and self.is_monotonic:
+            if not self.is_unique:
+                return self._get_loc_duplicates(val)
+            values = self._get_index_values()
+            loc = values.searchsorted(val, side='left')
+            if util.get_value_at(values, loc) != val:
+                raise KeyError(val)
+            return loc
+
         self._ensure_mapping_populated()
         if not self.unique:
             return self._get_loc_duplicates(val)
@@ -337,19 +355,17 @@ cdef class ObjectEngine(IndexEngine):
 
 cdef class DatetimeEngine(Int64Engine):
 
-    # cdef Int64HashTable mapping
-
     def __contains__(self, object val):
-        self._ensure_mapping_populated()
-
-        if util.is_datetime64_object(val):
-            return val.view('i8') in self.mapping
-
-        if PyDateTime_Check(val):
-            key = np.datetime64(val)
-            return key.view('i8') in self.mapping
+        if self.over_size_threshold and self.is_monotonic:
+            if not self.is_unique:
+                return self._get_loc_duplicates(val)
+            values = self._get_index_values()
+            conv = _to_i8(val)
+            loc = values.searchsorted(conv, side='left')
+            return util.get_value_at(values, loc) == conv
 
-        return val in self.mapping
+        self._ensure_mapping_populated()
+        return _to_i8(val) in self.mapping
 
     cdef _get_index_values(self):
         return self.index_weakref().values.view('i8')
@@ -363,13 +379,19 @@ cdef class DatetimeEngine(Int64Engine):
 
         # Welcome to the spaghetti factory
 
+        if self.over_size_threshold and self.is_monotonic:
+            if not self.is_unique:
+                return self._get_loc_duplicates(val)
+            values = self._get_index_values()
+            conv = _to_i8(val)
+            loc = values.searchsorted(conv, side='left')
+            if util.get_value_at(values, loc) != conv:
+                raise KeyError(val)
+            return loc
+
         self._ensure_mapping_populated()
         if not self.unique:
-            if util.is_datetime64_object(val):
-                val = val.view('i8')
-            elif PyDateTime_Check(val):
-                val = np.datetime64(val)
-                val = val.view('i8')
+            val = _to_i8(val)
             return self._get_loc_duplicates(val)
 
         try:
@@ -380,11 +402,7 @@ cdef class DatetimeEngine(Int64Engine):
             pass
 
         try:
-            if util.is_datetime64_object(val):
-                val = val.view('i8')
-            elif PyDateTime_Check(val):
-                val = np.datetime64(val)
-                val = val.view('i8')
+            val = _to_i8(val)
             return self.mapping.get_item(val)
         except TypeError:
             self._date_check_type(val)
@@ -417,6 +435,14 @@ cdef class DatetimeEngine(Int64Engine):
                                      limit=limit)
 
 
+cdef inline _to_i8(object val):
+    if util.is_datetime64_object(val):
+        val = unbox_datetime64_scalar(val)
+    elif PyDateTime_Check(val):
+        val = np.datetime64(val)
+        val = unbox_datetime64_scalar(val)
+    return val
+
 # ctypedef fused idxvalue_t:
 #     object
 #     int

vb_suite/timeseries.py

@@ -9,11 +9,24 @@
     rng = date_range('1/1/2000', periods=N, freq='min')
 except NameError:
     rng = DateRange('1/1/2000', periods=N, offset=datetools.Minute())
-    date_range = DateRange
+    def date_range(start=None, end=None, periods=None, freq=None):
+        return DateRange(start, end, periods=periods, offset=freq)
 
 ts = Series(np.random.randn(N), index=rng)
 """
 
+#----------------------------------------------------------------------
+# Lookup value in large time series, hash map population
+
+setup = common_setup + """
+rng = date_range('1/1/2000', periods=1500000, freq='s')
+ts = Series(1, index=rng)
+"""
+
+stmt = "ts[ts.index[len(ts) // 2]]; ts.index._cleanup()"
+timeseries_large_lookup_value = Benchmark(stmt, setup,
+                                          start_date=datetime(2012, 1, 1))
+
 #----------------------------------------------------------------------
 # Test slice minutely series