code cleanup

Author: jreback

asv_bench/benchmarks/reindex.py

@@ -16,8 +16,8 @@ def setup(self):
                              data=np.random.rand(10000, 30), columns=range(30))
 
         # multi-index
-        N = 1000
-        K = 20
+        N = 5000
+        K = 200
         level1 = tm.makeStringIndex(N).values.repeat(K)
         level2 = np.tile(tm.makeStringIndex(K).values, N)
         index = MultiIndex.from_arrays([level1, level2])

pandas/indexes/multi.py

@@ -667,14 +667,9 @@ def _has_complex_internals(self):
     @cache_readonly
     def is_monotonic(self):
 
-        def level_values(level):
-            unique = self.levels[level]
-            labels = self.labels[level]
-            return algos.take_1d(unique.values, labels,
-                                 fill_value=unique._na_value)
-
         # reversed() because lexsort() wants the most significant key last.
-        values = [level_values(i) for i in reversed(range(len(self.levels)))]
+        values = [self._get_level_values(i)
+                  for i in reversed(range(len(self.levels)))]
         try:
             sort_order = np.lexsort(values)
             return Index(sort_order).is_monotonic
@@ -827,26 +822,44 @@ def _try_mi(k):
 
             raise InvalidIndexError(key)
 
-    def get_level_values(self, level):
+    def _get_level_values(self, level):
         """
-        Return vector of label values for requested level, equal to the length
-        of the index
+        Return vector of label values for requested level,
+        equal to the length of the index
+
+        **this is an internal method**
 
         Parameters
         ----------
-        level : int or level name
+        level : int level
 
         Returns
         -------
         values : ndarray
         """
-        num = self._get_level_number(level)
-        unique = self.levels[num]  # .values
-        labels = self.labels[num]
-        filled = algos.take_1d(unique.values, labels,
+
+        unique = self.levels[level]
+        labels = self.labels[level]
+        filled = algos.take_1d(unique._values, labels,
                                fill_value=unique._na_value)
-        values = unique._shallow_copy(filled)
-        return values
+        return filled
+
+    def get_level_values(self, level):
+        """
+        Return vector of label values for requested level,
+        equal to the length of the index
+
+        Parameters
+        ----------
+        level : int or level name
+
+        Returns
+        -------
+        values : Index
+        """
+        level = self._get_level_number(level)
+        values = self._get_level_values(level)
+        return self.levels[level]._shallow_copy(values)
 
     def format(self, space=2, sparsify=None, adjoin=True, names=False,
                na_rep=None, formatter=None):

pandas/tools/hashing.py

@@ -5,7 +5,6 @@
 
 import numpy as np
 from pandas import _hash, Series, factorize, Categorical, Index, MultiIndex
-import pandas.core.algorithms as algos
 from pandas.lib import is_bool_array
 from pandas.types.generic import ABCIndexClass, ABCSeries, ABCDataFrame
 from pandas.types.common import (is_categorical_dtype, is_numeric_dtype,
@@ -115,7 +114,7 @@ def hash_pandas_object(obj, index=True, encoding='utf8', hash_key=None,
     return h
 
 
-def hash_tuples(vals, encoding='utf8', hash_key=None):
+def hash_tuples(vals, encoding='utf8', hash_key=None, categorize=True):
     """
     Hash an MultiIndex / list-of-tuples efficiently
 
@@ -126,6 +125,9 @@ def hash_tuples(vals, encoding='utf8', hash_key=None):
     vals : MultiIndex, list-of-tuples, or single tuple
     encoding : string, default 'utf8'
     hash_key : string key to encode, default to _default_hash_key
+    categorize : bool, default True
+        Whether to first categorize object arrays before hashing. This is more
+        efficient when the array contains duplicate values.
 
     Returns
     -------
@@ -143,18 +145,14 @@ def hash_tuples(vals, encoding='utf8', hash_key=None):
         vals = MultiIndex.from_tuples(vals)
 
     # create a list-of-ndarrays
-    def get_level_values(num):
-        unique = vals.levels[num]  # .values
-        labels = vals.labels[num]
-        filled = algos.take_1d(unique._values, labels,
-                               fill_value=unique._na_value)
-        return filled
-
-    vals = [get_level_values(level)
+    vals = [vals._get_level_values(level)
             for level in range(vals.nlevels)]
 
     # hash the list-of-ndarrays
-    hashes = (hash_array(l, encoding=encoding, hash_key=hash_key)
+    hashes = (hash_array(l,
+                         encoding=encoding,
+                         hash_key=hash_key,
+                         categorize=categorize)
               for l in vals)
     h = _combine_hash_arrays(hashes, len(vals))
     if is_tuple: