diff --git a/pandas/core/indexes/datetimelike.py b/pandas/core/indexes/datetimelike.py
index 5f24eb0cfaad6..484b581a898d8 100644
--- a/pandas/core/indexes/datetimelike.py
+++ b/pandas/core/indexes/datetimelike.py
@@ -632,10 +632,6 @@ def is_type_compatible(self, kind: str) -> bool:
     # --------------------------------------------------------------------
     # Set Operation Methods
 
-    def _difference(self, other, sort=None):
-        new_idx = super()._difference(other, sort=sort)._with_freq(None)
-        return new_idx
-
     def _intersection(self, other: Index, sort=False) -> Index:
         """
         intersection specialized to the case with matching dtypes.
@@ -781,13 +777,8 @@ def _union(self, other, sort):
 
         if self._can_fast_union(other):
             result = self._fast_union(other, sort=sort)
-            if sort is None:
-                # In the case where sort is None, _can_fast_union
-                #  implies that result.freq should match self.freq
-                assert result.freq == self.freq, (result.freq, self.freq)
-            elif result.freq is None:
-                # TODO: no tests rely on this; needed?
-                result = result._with_freq("infer")
+            # in the case with sort=None, the _can_fast_union check ensures
+            #  that result.freq == self.freq
             return result
         else:
             i8self = Int64Index._simple_new(self.asi8)