perf: Cach hashing of categories

Author: fjetter

doc/source/whatsnew/v1.3.0.rst

@@ -388,6 +388,7 @@ Performance improvements
 - Performance improvement in :class:`Styler` where render times are more than 50% reduced (:issue:`39972` :issue:`39952`)
 - Performance improvement in :meth:`core.window.ewm.ExponentialMovingWindow.mean` with ``times`` (:issue:`39784`)
 - Performance improvement in :meth:`.GroupBy.apply` when requiring the python fallback implementation (:issue:`40176`)
+- Performance improvement for concatenation of data with type :class:`CategoricalDtype` (:issue:`40193`)
 
 .. ---------------------------------------------------------------------------
 

pandas/core/dtypes/dtypes.py

@@ -21,6 +21,7 @@
 import pytz
 
 from pandas._libs.interval import Interval
+from pandas._libs.properties import cache_readonly
 from pandas._libs.tslibs import (
     BaseOffset,
     NaT,
@@ -351,17 +352,6 @@ def __setstate__(self, state: MutableMapping[str_type, Any]) -> None:
         self._categories = state.pop("categories", None)
         self._ordered = state.pop("ordered", False)
 
-    def __hash__(self) -> int:
-        # _hash_categories returns a uint64, so use the negative
-        # space for when we have unknown categories to avoid a conflict
-        if self.categories is None:
-            if self.ordered:
-                return -1
-            else:
-                return -2
-        # We *do* want to include the real self.ordered here
-        return int(self._hash_categories(self.categories, self.ordered))
-
     def __eq__(self, other: Any) -> bool:
         """
         Rules for CDT equality:
@@ -434,14 +424,29 @@ def __repr__(self) -> str_type:
             data = data.rstrip(", ")
         return f"CategoricalDtype(categories={data}, ordered={self.ordered})"
 
-    @staticmethod
-    def _hash_categories(categories, ordered: Ordered = True) -> int:
+    def __hash__(self) -> int:
+        # _hash_categories returns a uint64, so use the negative
+        # space for when we have unknown categories to avoid a conflict
+        if self.categories is None:
+            if self.ordered:
+                return -1
+            else:
+                return -2
+        # Indirection via property to allow for easier caching
+        return self._hash_categories
+
+    @cache_readonly
+    def _hash_categories(self) -> int:
         from pandas.core.util.hashing import (
             combine_hash_arrays,
             hash_array,
             hash_tuples,
         )
 
+        # We *do* want to include the real self.ordered here
+        categories = self.categories
+        ordered = self.ordered
+
         if len(categories) and isinstance(categories[0], tuple):
             # assumes if any individual category is a tuple, then all our. ATM
             # I don't really want to support just some of the categories being
@@ -469,7 +474,7 @@ def _hash_categories(categories, ordered: Ordered = True) -> int:
         else:
             cat_array = [cat_array]
         hashed = combine_hash_arrays(iter(cat_array), num_items=len(cat_array))
-        return np.bitwise_xor.reduce(hashed)
+        return int(np.bitwise_xor.reduce(hashed))
 
     @classmethod
     def construct_array_type(cls) -> Type[Categorical]: