diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py
index 8ebaaa28e13a5..c4a3afbb282cf 100644
--- a/pandas/core/indexes/base.py
+++ b/pandas/core/indexes/base.py
@@ -4205,9 +4205,14 @@ def is_int(v):
             return v is None or is_integer(v)
 
         is_index_slice = is_int(start) and is_int(stop) and is_int(step)
-        is_positional = is_index_slice and not (
-            self.is_integer() or self.is_categorical()
+
+        # special case for interval_dtype bc we do not do partial-indexing
+        #  on integer Intervals when slicing
+        # TODO: write this in terms of e.g. should_partial_index?
+        ints_are_positional = self._should_fallback_to_positional or is_interval_dtype(
+            self.dtype
         )
+        is_positional = is_index_slice and ints_are_positional
 
         if kind == "getitem":
             """
diff --git a/pandas/core/indexes/numeric.py b/pandas/core/indexes/numeric.py
index 174e0a7f81850..c1cb5ad315298 100644
--- a/pandas/core/indexes/numeric.py
+++ b/pandas/core/indexes/numeric.py
@@ -222,6 +222,8 @@ def _convert_slice_indexer(self, key: slice, kind: str, is_frame: bool = False):
         if is_float_dtype(self.dtype):
             assert kind in ["loc", "getitem"]
 
+            # TODO: can we write this as a condition based on
+            #  e.g. _should_fallback_to_positional?
             # We always treat __getitem__ slicing as label-based
             # translate to locations
             return self.slice_indexer(key.start, key.stop, key.step)
diff --git a/pandas/core/indexing.py b/pandas/core/indexing.py
index ddae58fd46bb0..02c095202d079 100644
--- a/pandas/core/indexing.py
+++ b/pandas/core/indexing.py
@@ -1322,10 +1322,6 @@ def _convert_to_indexer(self, key, axis: int):
         if isinstance(key, slice):
             return labels._convert_slice_indexer(key, kind="loc")
 
-        # see if we are positional in nature
-        is_int_index = labels.is_integer()
-        is_int_positional = is_integer(key) and not is_int_index
-
         if (
             isinstance(key, tuple)
             and not isinstance(labels, MultiIndex)
@@ -1350,17 +1346,9 @@ def _convert_to_indexer(self, key, axis: int):
                 if not isinstance(labels, MultiIndex):
                     raise
             except ValueError:
-                if not is_int_positional:
+                if not is_integer(key):
                     raise
-
-        # a positional
-        if is_int_positional:
-
-            # if we are setting and its not a valid location
-            # its an insert which fails by definition
-
-            # always valid
-            return {"key": key}
+                return {"key": key}
 
         if is_nested_tuple(key, labels):
             if self.ndim == 1 and any(isinstance(k, tuple) for k in key):