BUG: Categorical.searchsorted(): use provided categorical order

Previously, it used lexical order instead of the provided categorical
order.

Tests updated accordingly.

Closes #14522

Author: nathalier

pandas/core/categorical.py

@@ -1085,10 +1085,14 @@ def searchsorted(self, v, side='left', sorter=None):
                              "ordered one")
 
         from pandas.core.series import Series
-        values_as_codes = self.categories.values.searchsorted(
-            Series(v).values, side=side)
+        values_as_codes = _get_codes_for_values(Series(v).values,
+                                                self.categories)
 
-        return self.codes.searchsorted(values_as_codes, sorter=sorter)
+        if -1 in values_as_codes:
+            raise ValueError("Value(s) to be inserted must be in categories.")
+
+        return self.codes.searchsorted(values_as_codes, side=side,
+                                       sorter=sorter)
 
     def isnull(self):
         """

pandas/tests/test_categorical.py

@@ -1548,50 +1548,47 @@ def test_memory_usage(self):
 
     def test_searchsorted(self):
         # https://github.com/pandas-dev/pandas/issues/8420
-        s1 = pd.Series(['apple', 'bread', 'bread', 'cheese', 'milk'])
-        s2 = pd.Series(['apple', 'bread', 'bread', 'cheese', 'milk', 'donuts'])
-        c1 = pd.Categorical(s1, ordered=True)
-        c2 = pd.Categorical(s2, ordered=True)
-
-        # Single item array
-        res = c1.searchsorted(['bread'])
-        chk = s1.searchsorted(['bread'])
-        exp = np.array([1], dtype=np.intp)
-        self.assert_numpy_array_equal(res, exp)
-        self.assert_numpy_array_equal(res, chk)
-
-        # Scalar version of single item array
-        # Categorical return np.array like pd.Series, but different from
-        # np.array.searchsorted()
-        res = c1.searchsorted('bread')
-        chk = s1.searchsorted('bread')
-        exp = np.array([1], dtype=np.intp)
+        # https://github.com/pandas-dev/pandas/issues/14522
+
+        c1 = pd.Categorical(['cheese', 'apple', 'bread', 'bread', 'milk'],
+                 categories=['cheese', 'milk', 'apple', 'bread'],
+                 ordered=True).sort_values()
+        c2 = pd.Categorical(['cheese', 'apple', 'bread', 'bread',  'milk'],
+                 categories=['cheese', 'milk', 'apple', 'bread'],
+                 ordered=False).sort_values()
+
+        # Searching for single item argument, side='left' (default)
+        res = c1.searchsorted('apple')
+        exp = np.array([2], dtype=np.intp)
         self.assert_numpy_array_equal(res, exp)
-        self.assert_numpy_array_equal(res, chk)
 
-        # Searching for a value that is not present in the Categorical
-        res = c1.searchsorted(['bread', 'eggs'])
-        chk = s1.searchsorted(['bread', 'eggs'])
-        exp = np.array([1, 4], dtype=np.intp)
+        # Searching for single item array, side='left' (default)
+        res = c1.searchsorted(['bread'])
+        exp = np.array([3], dtype=np.intp)
         self.assert_numpy_array_equal(res, exp)
-        self.assert_numpy_array_equal(res, chk)
 
-        # Searching for a value that is not present, to the right
-        res = c1.searchsorted(['bread', 'eggs'], side='right')
-        chk = s1.searchsorted(['bread', 'eggs'], side='right')
-        exp = np.array([3, 4], dtype=np.intp)  # eggs before milk
-        self.assert_numpy_array_equal(res, exp)
-        self.assert_numpy_array_equal(res, chk)
-
-        # As above, but with a sorter array to reorder an unsorted array
-        res = c2.searchsorted(['bread', 'eggs'], side='right',
-                              sorter=[0, 1, 2, 3, 5, 4])
-        chk = s2.searchsorted(['bread', 'eggs'], side='right',
-                              sorter=[0, 1, 2, 3, 5, 4])
-        # eggs after donuts, after switching milk and donuts
+        # Searching for several items array, side='right'
+        res = c1.searchsorted(['apple', 'bread'], side='right')
         exp = np.array([3, 5], dtype=np.intp)
         self.assert_numpy_array_equal(res, exp)
-        self.assert_numpy_array_equal(res, chk)
+
+        # Searching for a single value that is not from the Categorical
+        def f():
+            c1.searchsorted('cucumber')
+
+        self.assertRaises(ValueError, f)
+
+        # Searching for multiple values one of each is not from the Categorical
+        def f():
+            c1.searchsorted(['bread', 'cucumber'])
+
+        self.assertRaises(ValueError, f)
+
+        # searchsorted call for not ordered Categorical
+        def f():
+            c2.searchsorted('apple')
+
+        self.assertRaises(ValueError, f)
 
     def test_deprecated_labels(self):
         # TODO: labels is deprecated and should be removed in 0.18 or 2017,