diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py
index 3cfa1f15fa118..53b5bf8f182bd 100644
--- a/pandas/core/indexes/base.py
+++ b/pandas/core/indexes/base.py
@@ -3209,7 +3209,6 @@ def _wrap_difference_result(self, other, result):
         # We will override for MultiIndex to handle empty results
         return self._wrap_setop_result(other, result)
 
-    @final
     def symmetric_difference(self, other, result_name=None, sort=None):
         """
         Compute the symmetric difference of two Index objects.
@@ -3251,21 +3250,46 @@ def symmetric_difference(self, other, result_name=None, sort=None):
         if result_name is None:
             result_name = result_name_update
 
-        left = self.difference(other, sort=False)
-        right = other.difference(self, sort=False)
-        result = left.union(right, sort=sort)
+        if not self._should_compare(other):
+            return self.union(other, sort=sort).rename(result_name)
 
-        if result_name is not None:
-            result = result.rename(result_name)
+        elif not is_dtype_equal(self.dtype, other.dtype):
+            dtype = self._find_common_type_compat(other)
+            this = self.astype(dtype, copy=False)
+            that = other.astype(dtype, copy=False)
+            return this.symmetric_difference(that, sort=sort).rename(result_name)
 
-        if self._is_multi and len(result) == 0:
-            # On equal symmetric_difference MultiIndexes the difference is empty.
-            # Therefore, an empty MultiIndex is returned GH#13490
-            return type(self)(
-                levels=[[] for _ in range(self.nlevels)],
-                codes=[[] for _ in range(self.nlevels)],
-                names=result.names,
-            )
+        this = self.unique()
+        other = other.unique()
+        indexer = this.get_indexer_for(other)
+
+        # {this} minus {other}
+        common_indexer = indexer.take((indexer != -1).nonzero()[0])
+        left_indexer = np.setdiff1d(
+            np.arange(this.size), common_indexer, assume_unique=True
+        )
+        left_diff = this._values.take(left_indexer)
+
+        # {other} minus {this}
+        right_indexer = (indexer == -1).nonzero()[0]
+        right_diff = other._values.take(right_indexer)
+
+        res_values = concat_compat([left_diff, right_diff])
+        res_values = _maybe_try_sort(res_values, sort)
+
+        result = Index(res_values, name=result_name)
+
+        if self._is_multi:
+            self = cast("MultiIndex", self)
+            if len(result) == 0:
+                # On equal symmetric_difference MultiIndexes the difference is empty.
+                # Therefore, an empty MultiIndex is returned GH#13490
+                return type(self)(
+                    levels=[[] for _ in range(self.nlevels)],
+                    codes=[[] for _ in range(self.nlevels)],
+                    names=result.name,
+                )
+            return type(self).from_tuples(result, names=result.name)
 
         return result
 
diff --git a/pandas/core/indexes/datetimelike.py b/pandas/core/indexes/datetimelike.py
index 92755ac02c46d..8da99961a0a24 100644
--- a/pandas/core/indexes/datetimelike.py
+++ b/pandas/core/indexes/datetimelike.py
@@ -64,6 +64,7 @@
     NDArrayBackedExtensionIndex,
     inherit_names,
 )
+from pandas.core.indexes.numeric import Int64Index
 from pandas.core.tools.timedeltas import to_timedelta
 
 if TYPE_CHECKING:
@@ -680,4 +681,8 @@ def _union(self, other, sort):
             #  that result.freq == self.freq
             return result
         else:
-            return super()._union(other, sort=sort)._with_freq("infer")
+            i8self = Int64Index._simple_new(self.asi8)
+            i8other = Int64Index._simple_new(other.asi8)
+            i8result = i8self._union(i8other, sort=sort)
+            result = type(self)(i8result, dtype=self.dtype, freq="infer")
+            return result
diff --git a/pandas/core/indexes/range.py b/pandas/core/indexes/range.py
index 8bef105c783c8..41997ad7ca46e 100644
--- a/pandas/core/indexes/range.py
+++ b/pandas/core/indexes/range.py
@@ -731,6 +731,18 @@ def _difference(self, other, sort=None):
             new_index = new_index[::-1]
         return new_index
 
+    def symmetric_difference(self, other, result_name: Hashable = None, sort=None):
+        if not isinstance(other, RangeIndex) or sort is not None:
+            return super().symmetric_difference(other, result_name, sort)
+
+        left = self.difference(other)
+        right = other.difference(self)
+        result = left.union(right)
+
+        if result_name is not None:
+            result = result.rename(result_name)
+        return result
+
     # --------------------------------------------------------------------
 
     def _concat(self, indexes: list[Index], name: Hashable) -> Index: