PERF: fix iteritems performance regression in unique case. #2336

Author: wesm

pandas/core/algorithms.py

@@ -56,13 +56,13 @@ def unique(values):
     return _hashtable_algo(f, values.dtype)
 
 
-def count(values, uniques=None):
-    f = lambda htype, caster: _count_generic(values, htype, caster)
+# def count(values, uniques=None):
+#     f = lambda htype, caster: _count_generic(values, htype, caster)
 
-    if uniques is not None:
-        raise NotImplementedError
-    else:
-        return _hashtable_algo(f, values.dtype)
+#     if uniques is not None:
+#         raise NotImplementedError
+#     else:
+#         return _hashtable_algo(f, values.dtype)
 
 
 def _hashtable_algo(f, dtype):
@@ -82,7 +82,7 @@ def _count_generic(values, table_type, type_caster):
 
     values = type_caster(values)
     table = table_type(min(len(values), 1000000))
-    uniques, labels, counts = table.factorize(values)
+    uniques, labels = table.factorize(values)
 
     return Series(counts, index=uniques)
 

pandas/core/frame.py

@@ -681,8 +681,12 @@ def keys(self):
 
     def iteritems(self):
         """Iterator over (column, series) pairs"""
-        for i, k in enumerate(self.columns):
-            yield k, self.icol(i)
+        if self.columns.is_unique and hasattr(self, '_item_cache'):
+            for k in self.columns:
+                yield k, self._get_item_cache(k)
+        else:
+            for i, k in enumerate(self.columns):
+                yield k, self.icol(i)
 
     def iterrows(self):
         """
@@ -1829,15 +1833,8 @@ def icol(self, i):
                     return self.ix[:, i]
 
             values = self._data.iget(i)
-            if hasattr(self,'default_fill_value'):
-                s = self._col_klass.from_array(values, index=self.index,
-                                               name=label,
-                                               fill_value= self.default_fill_value)
-            else:
-                s = self._col_klass.from_array(values, index=self.index,
-                                               name=label)
-
-            return s
+            return self._col_klass.from_array(values, index=self.index,
+                                              name=label)
 
     def _ixs(self, i, axis=0):
         if axis == 0:

pandas/sparse/frame.py

@@ -377,6 +377,10 @@ def icol(self, i):
                     return self.ix[:, i]
 
             return self[label]
+            # values = self._data.iget(i)
+            # return self._col_klass.from_array(
+            #     values, index=self.index, name=label,
+            #     fill_value= self.default_fill_value)
 
     @Appender(DataFrame.get_value.__doc__, indents=0)
     def get_value(self, index, col):

pandas/src/hashtable.pyx

@@ -444,24 +444,6 @@ cdef class Int64HashTable(HashTable):
 
         return locs
 
-    def lookup_i4(self, ndarray[int64_t] values):
-        cdef:
-            Py_ssize_t i, n = len(values)
-            int ret = 0
-            int64_t val
-            khiter_t k
-            ndarray[int64_t] locs = np.empty(n, dtype=np.int64)
-
-        for i in range(n):
-            val = values[i]
-            k = kh_get_int64(self.table, val)
-            if k != self.table.n_buckets:
-                locs[i] = self.table.vals[k]
-            else:
-                locs[i] = -1
-
-        return locs
-
     def factorize(self, ndarray[object] values):
         reverse = {}
         labels = self.get_labels(values, reverse, 0)

vb_suite/frame_methods.py

@@ -71,8 +71,20 @@
 
 setup = common_setup + """
 df = DataFrame(randn(10000, 100))
+def f():
+    if hasattr(df, '_item_cache'):
+        df._item_cache.clear()
+    for name, col in df.iteritems():
+        pass
+
+def g():
+    for name, col in df.iteritems():
+        pass
 """
 
 # as far back as the earliest test currently in the suite
-frame_iteritems = Benchmark('for name,col in df.iteritems(): pass', setup,
-                                     start_date=datetime(2010, 6, 1))
+frame_iteritems = Benchmark('f()', setup,
+                            start_date=datetime(2010, 6, 1))
+
+frame_iteritems_cached = Benchmark('g()', setup,
+                                   start_date=datetime(2010, 6, 1))