diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py
index 9a669a19053b0..6c33510ddf1bc 100644
--- a/pandas/core/indexes/base.py
+++ b/pandas/core/indexes/base.py
@@ -163,7 +163,6 @@
 )
 
 if TYPE_CHECKING:
-
     from pandas import (
         CategoricalIndex,
         DataFrame,
@@ -172,6 +171,7 @@
         RangeIndex,
         Series,
     )
+    from pandas.core.arrays import PeriodArray
 
 
 __all__ = ["Index"]
@@ -3701,19 +3701,18 @@ def _get_fill_indexer(
                 target=target._values, values=self._values, method=method, limit=limit
             )
 
-        target_values = target._get_engine_target()
-
         if self.is_monotonic_increasing and target.is_monotonic_increasing:
             engine_method = (
                 self._engine.get_pad_indexer
                 if method == "pad"
                 else self._engine.get_backfill_indexer
             )
+            target_values = target._get_engine_target()
             indexer = engine_method(target_values, limit)
         else:
             indexer = self._get_fill_indexer_searchsorted(target, method, limit)
         if tolerance is not None and len(self):
-            indexer = self._filter_indexer_tolerance(target_values, indexer, tolerance)
+            indexer = self._filter_indexer_tolerance(target, indexer, tolerance)
         return indexer
 
     @final
@@ -3765,10 +3764,8 @@ def _get_nearest_indexer(
         left_indexer = self.get_indexer(target, "pad", limit=limit)
         right_indexer = self.get_indexer(target, "backfill", limit=limit)
 
-        target_values = target._get_engine_target()
-        own_values = self._get_engine_target()
-        left_distances = np.abs(own_values[left_indexer] - target_values)
-        right_distances = np.abs(own_values[right_indexer] - target_values)
+        left_distances = self._difference_compat(target, left_indexer)
+        right_distances = self._difference_compat(target, right_indexer)
 
         op = operator.lt if self.is_monotonic_increasing else operator.le
         indexer = np.where(
@@ -3777,20 +3774,39 @@ def _get_nearest_indexer(
             right_indexer,
         )
         if tolerance is not None:
-            indexer = self._filter_indexer_tolerance(target_values, indexer, tolerance)
+            indexer = self._filter_indexer_tolerance(target, indexer, tolerance)
         return indexer
 
     @final
     def _filter_indexer_tolerance(
         self,
-        target: np.ndarray,
+        target: Index,
         indexer: npt.NDArray[np.intp],
         tolerance,
     ) -> npt.NDArray[np.intp]:
-        own_values = self._get_engine_target()
-        distance = abs(own_values[indexer] - target)
+
+        distance = self._difference_compat(target, indexer)
+
         return np.where(distance <= tolerance, indexer, -1)
 
+    @final
+    def _difference_compat(
+        self, target: Index, indexer: npt.NDArray[np.intp]
+    ) -> ArrayLike:
+        # Compatibility for PeriodArray, for which __sub__ returns an ndarray[object]
+        #  of DateOffset objects, which do not support __abs__ (and would be slow
+        #  if they did)
+
+        if isinstance(self.dtype, PeriodDtype):
+            # Note: we only get here with matching dtypes
+            own_values = cast("PeriodArray", self._data)._ndarray
+            target_values = cast("PeriodArray", target._data)._ndarray
+            diff = own_values[indexer] - target_values
+        else:
+            # error: Unsupported left operand type for - ("ExtensionArray")
+            diff = self._values[indexer] - target._values  # type: ignore[operator]
+        return abs(diff)
+
     # --------------------------------------------------------------------
     # Indexer Conversion Methods