diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py
index 52f3c14e59184..0be3970159fbd 100644
--- a/pandas/core/dtypes/cast.py
+++ b/pandas/core/dtypes/cast.py
@@ -1952,4 +1952,7 @@ def can_hold_element(dtype: np.dtype, element: Any) -> bool:
             return tipo.kind == "b"
         return lib.is_bool(element)
 
+    elif dtype == object:
+        return True
+
     raise NotImplementedError(dtype)
diff --git a/pandas/core/dtypes/common.py b/pandas/core/dtypes/common.py
index 9861a466b2d2f..dc6d0784df217 100644
--- a/pandas/core/dtypes/common.py
+++ b/pandas/core/dtypes/common.py
@@ -1677,7 +1677,7 @@ def _is_dtype_type(arr_or_dtype, condition) -> bool:
     return condition(tipo)
 
 
-def infer_dtype_from_object(dtype):
+def infer_dtype_from_object(dtype) -> DtypeObj:
     """
     Get a numpy dtype.type-style object for a dtype object.
 
diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py
index 40215ea87f978..98fbb9c761b47 100644
--- a/pandas/core/indexes/base.py
+++ b/pandas/core/indexes/base.py
@@ -40,6 +40,7 @@
 from pandas.util._decorators import Appender, cache_readonly, doc
 
 from pandas.core.dtypes.cast import (
+    can_hold_element,
     find_common_type,
     maybe_cast_to_integer_array,
     maybe_promote,
@@ -4360,6 +4361,8 @@ def _validate_fill_value(self, value):
         TypeError
             If the value cannot be inserted into an array of this dtype.
         """
+        if not can_hold_element(self.dtype, value):
+            raise TypeError
         return value
 
     @final
diff --git a/pandas/core/indexes/datetimelike.py b/pandas/core/indexes/datetimelike.py
index dca3a4458bc68..55317938d528d 100644
--- a/pandas/core/indexes/datetimelike.py
+++ b/pandas/core/indexes/datetimelike.py
@@ -609,6 +609,12 @@ def insert(self, loc: int, item):
         result._data._freq = self._get_insert_freq(loc, item)
         return result
 
+    def _validate_fill_value(self, value):
+        """
+        Convert value to be insertable to ndarray.
+        """
+        return self._data._validate_setitem_value(value)
+
     # --------------------------------------------------------------------
     # Join/Set Methods
 
diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py
index abe9a9c26c663..a6d71cadddd64 100644
--- a/pandas/core/indexes/datetimes.py
+++ b/pandas/core/indexes/datetimes.py
@@ -342,12 +342,6 @@ def __reduce__(self):
         d.update(self._get_attributes_dict())
         return _new_DatetimeIndex, (type(self), d), None
 
-    def _validate_fill_value(self, value):
-        """
-        Convert value to be insertable to ndarray.
-        """
-        return self._data._validate_setitem_value(value)
-
     def _is_comparable_dtype(self, dtype: DtypeObj) -> bool:
         """
         Can we compare values of the given dtype to our own?
diff --git a/pandas/core/indexes/numeric.py b/pandas/core/indexes/numeric.py
index 26599bd6ab871..a432b3952666e 100644
--- a/pandas/core/indexes/numeric.py
+++ b/pandas/core/indexes/numeric.py
@@ -10,13 +10,11 @@
 from pandas.core.dtypes.cast import astype_nansafe
 from pandas.core.dtypes.common import (
     is_bool,
-    is_bool_dtype,
     is_dtype_equal,
     is_extension_array_dtype,
     is_float,
     is_float_dtype,
     is_integer_dtype,
-    is_number,
     is_numeric_dtype,
     is_scalar,
     is_signed_integer_dtype,
@@ -25,7 +23,6 @@
     pandas_dtype,
 )
 from pandas.core.dtypes.generic import ABCSeries
-from pandas.core.dtypes.missing import is_valid_nat_for_dtype, isna
 
 import pandas.core.common as com
 from pandas.core.indexes.base import Index, maybe_extract_name
@@ -122,42 +119,6 @@ def _shallow_copy(self, values=None, name: Hashable = lib.no_default):
             return Float64Index._simple_new(values, name=name)
         return super()._shallow_copy(values=values, name=name)
 
-    @doc(Index._validate_fill_value)
-    def _validate_fill_value(self, value):
-        if is_bool(value) or is_bool_dtype(value):
-            # force conversion to object
-            # so we don't lose the bools
-            raise TypeError
-        elif is_scalar(value) and isna(value):
-            if is_valid_nat_for_dtype(value, self.dtype):
-                value = self._na_value
-                if self.dtype.kind != "f":
-                    # raise so that caller can cast
-                    raise TypeError
-            else:
-                # NaT, np.datetime64("NaT"), np.timedelta64("NaT")
-                raise TypeError
-
-        elif is_scalar(value):
-            if not is_number(value):
-                # e.g. datetime64, timedelta64, datetime, ...
-                raise TypeError
-
-            elif lib.is_complex(value):
-                # at least until we have a ComplexIndx
-                raise TypeError
-
-            elif is_float(value) and self.dtype.kind != "f":
-                if not value.is_integer():
-                    raise TypeError
-                value = int(value)
-
-        elif hasattr(value, "dtype") and value.dtype.kind in ["m", "M"]:
-            # TODO: if we're checking arraylike here, do so systematically
-            raise TypeError
-
-        return value
-
     def _convert_tolerance(self, tolerance, target):
         tolerance = np.asarray(tolerance)
         if target.size != tolerance.size and tolerance.size > 1: