diff --git a/pandas/_libs/index.pyx b/pandas/_libs/index.pyx
index 1915eaf6e07dd..b39afc57f34f6 100644
--- a/pandas/_libs/index.pyx
+++ b/pandas/_libs/index.pyx
@@ -87,20 +87,6 @@ cdef class IndexEngine:
         else:
             return get_value_at(arr, loc, tz=tz)
 
-    cpdef set_value(self, ndarray arr, object key, object value):
-        """
-        Parameters
-        ----------
-        arr : 1-dimensional ndarray
-        """
-        cdef:
-            object loc
-
-        loc = self.get_loc(key)
-        value = convert_scalar(arr, value)
-
-        arr[loc] = value
-
     cpdef get_loc(self, object val):
         cdef:
             Py_ssize_t loc
@@ -585,16 +571,23 @@ cpdef convert_scalar(ndarray arr, object value):
         raise ValueError("cannot set a Timedelta with a non-timedelta "
                          f"{type(value).__name__}")
 
-    if (issubclass(arr.dtype.type, (np.integer, np.floating, np.complex)) and
-            not issubclass(arr.dtype.type, np.bool_)):
-        if util.is_bool_object(value):
-            raise ValueError("Cannot assign bool to float/integer series")
+    else:
+        validate_numeric_casting(arr.dtype, value)
+
+    return value
+
 
-    if issubclass(arr.dtype.type, (np.integer, np.bool_)):
+cpdef validate_numeric_casting(dtype, object value):
+    # Note: we can't annotate dtype as cnp.dtype because that cases dtype.type
+    #  to integer
+    if issubclass(dtype.type, (np.integer, np.bool_)):
         if util.is_float_object(value) and value != value:
             raise ValueError("Cannot assign nan to integer series")
 
-    return value
+    if (issubclass(dtype.type, (np.integer, np.floating, np.complex)) and
+            not issubclass(dtype.type, np.bool_)):
+        if util.is_bool_object(value):
+            raise ValueError("Cannot assign bool to float/integer series")
 
 
 cdef class BaseMultiIndexCodesEngine:
diff --git a/pandas/core/frame.py b/pandas/core/frame.py
index 0dea8235e9d3f..7ea4ece76201d 100644
--- a/pandas/core/frame.py
+++ b/pandas/core/frame.py
@@ -40,7 +40,7 @@
 
 from pandas._config import get_option
 
-from pandas._libs import algos as libalgos, lib, properties
+from pandas._libs import algos as libalgos, index as libindex, lib, properties
 from pandas._typing import Axes, Axis, Dtype, FilePathOrBuffer, Label, Level, Renamer
 from pandas.compat import PY37
 from pandas.compat._optional import import_optional_dependency
@@ -3028,10 +3028,14 @@ def _set_value(self, index, col, value, takeable: bool = False):
 
             series = self._get_item_cache(col)
             engine = self.index._engine
-            engine.set_value(series._values, index, value)
+            loc = engine.get_loc(index)
+            libindex.validate_numeric_casting(series.dtype, value)
+
+            series._values[loc] = value
+            # Note: trying to use series._set_value breaks tests in
+            #  tests.frame.indexing.test_indexing and tests.indexing.test_partial
             return self
         except (KeyError, TypeError):
-
             # set using a non-recursive method & reset the cache
             if takeable:
                 self.iloc[index, col] = value
diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py
index 5e6018d85bd2d..5cac95c6702e1 100644
--- a/pandas/core/indexes/base.py
+++ b/pandas/core/indexes/base.py
@@ -4650,9 +4650,9 @@ def set_value(self, arr, key, value):
             FutureWarning,
             stacklevel=2,
         )
-        self._engine.set_value(
-            com.values_from_object(arr), com.values_from_object(key), value
-        )
+        loc = self._engine.get_loc(key)
+        libindex.validate_numeric_casting(arr.dtype, value)
+        arr[loc] = value
 
     _index_shared_docs[
         "get_indexer_non_unique"
diff --git a/pandas/core/series.py b/pandas/core/series.py
index e5cea8ebfc914..8ad0fe8f705b1 100644
--- a/pandas/core/series.py
+++ b/pandas/core/series.py
@@ -1026,17 +1026,10 @@ def __setitem__(self, key, value):
             self._maybe_update_cacher()
 
     def _set_with_engine(self, key, value):
-        values = self._values
-        if is_extension_array_dtype(values.dtype):
-            # The cython indexing engine does not support ExtensionArrays.
-            values[self.index.get_loc(key)] = value
-            return
-        try:
-            self.index._engine.set_value(values, key, value)
-            return
-        except KeyError:
-            values[self.index.get_loc(key)] = value
-            return
+        # fails with AttributeError for IntervalIndex
+        loc = self.index._engine.get_loc(key)
+        libindex.validate_numeric_casting(self.dtype, value)
+        self._values[loc] = value
 
     def _set_with(self, key, value):
         # other: fancy integer or otherwise
@@ -1116,11 +1109,10 @@ def _set_value(self, label, value, takeable: bool = False):
         try:
             if takeable:
                 self._values[label] = value
-            elif isinstance(self._values, np.ndarray):
-                # i.e. not EA, so we can use _engine
-                self.index._engine.set_value(self._values, label, value)
             else:
-                self.loc[label] = value
+                loc = self.index.get_loc(label)
+                libindex.validate_numeric_casting(self.dtype, value)
+                self._values[loc] = value
         except KeyError:
 
             # set using a non-recursive method
diff --git a/pandas/tests/indexing/test_indexing.py b/pandas/tests/indexing/test_indexing.py
index 1913caae93932..ae32274c02dcd 100644
--- a/pandas/tests/indexing/test_indexing.py
+++ b/pandas/tests/indexing/test_indexing.py
@@ -137,7 +137,7 @@ def test_setitem_ndarray_3d(self, index, obj, idxr, idxr_id):
             r"Buffer has wrong number of dimensions \(expected 1, "
             r"got 3\)|"
             "'pandas._libs.interval.IntervalTree' object has no attribute "
-            "'set_value'|"  # AttributeError
+            "'get_loc'|"  # AttributeError
             "unhashable type: 'numpy.ndarray'|"  # TypeError
             "No matching signature found|"  # TypeError
             r"^\[\[\[|"  # pandas.core.indexing.IndexingError