diff --git a/pandas/tseries/frequencies.py b/pandas/tseries/frequencies.py
index 522daf7188bfb..391d076acf68f 100644
--- a/pandas/tseries/frequencies.py
+++ b/pandas/tseries/frequencies.py
@@ -1,6 +1,9 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING
+from typing import (
+    TYPE_CHECKING,
+    cast,
+)
 
 import numpy as np
 
@@ -51,6 +54,7 @@
         Series,
         TimedeltaIndex,
     )
+    from pandas.core.arrays import TimedeltaArray
     from pandas.core.arrays.datetimelike import DatetimeLikeArrayMixin
 # ---------------------------------------------------------------------
 # Offset names ("time rules") and related functions
@@ -140,10 +144,7 @@ def infer_freq(
     >>> pd.infer_freq(idx)
     'D'
     """
-    from pandas.core.api import (
-        DatetimeIndex,
-        Index,
-    )
+    from pandas.core.api import DatetimeIndex
 
     if isinstance(index, ABCSeries):
         values = index._values
@@ -172,15 +173,14 @@ def infer_freq(
         inferer = _TimedeltaFrequencyInferer(index)
         return inferer.get_freq()
 
-    if isinstance(index, Index) and not isinstance(index, DatetimeIndex):
+    if not hasattr(index, "dtype"):
+        pass
+    elif isinstance(index.dtype, object):
         if is_numeric_dtype(index.dtype):
             raise TypeError(
                 f"cannot infer freq from a non-convertible index of dtype {index.dtype}"
             )
-        # error: Incompatible types in assignment (expression has type
-        # "Union[ExtensionArray, ndarray[Any, Any]]", variable has type
-        # "Union[DatetimeIndex, TimedeltaIndex, Series, DatetimeLikeArrayMixin]")
-        index = index._values  # type: ignore[assignment]
+        index = cast("TimedeltaArray", index)
 
     if not isinstance(index, DatetimeIndex):
         index = DatetimeIndex(index)