diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py
index ee2199112b65a..83849ea41d032 100644
--- a/pandas/core/internals/blocks.py
+++ b/pandas/core/internals/blocks.py
@@ -739,6 +739,13 @@ def _try_coerce_args(self, other):
                     type(self).__name__.lower().replace("Block", ""),
                 )
             )
+        if np.any(isna(other)) and not self._can_hold_na:
+            raise TypeError(
+                "cannot convert {} to an {}".format(
+                    type(other).__name__,
+                    type(self).__name__.lower().replace("Block", ""),
+                )
+            )
 
         return other
 
@@ -787,16 +794,15 @@ def replace(
         inplace = validate_bool_kwarg(inplace, "inplace")
         original_to_replace = to_replace
 
-        # try to replace, if we raise an error, convert to ObjectBlock and
+        # If we cannot replace with own dtype, convert to ObjectBlock and
         # retry
-        values = self._coerce_values(self.values)
-        try:
-            to_replace = self._try_coerce_args(to_replace)
-        except (TypeError, ValueError):
+        if not self._can_hold_element(to_replace):
+            # TODO: we should be able to infer at this point that there is
+            #  nothing to replace
             # GH 22083, TypeError or ValueError occurred within error handling
             # causes infinite loop. Cast and retry only if not objectblock.
             if is_object_dtype(self):
-                raise
+                raise AssertionError
 
             # try again with a compatible block
             block = self.astype(object)
@@ -809,6 +815,9 @@ def replace(
                 convert=convert,
             )
 
+        values = self._coerce_values(self.values)
+        to_replace = self._try_coerce_args(to_replace)
+
         mask = missing.mask_missing(values, to_replace)
         if filter is not None:
             filtered_out = ~self.mgr_locs.isin(filter)
@@ -1405,7 +1414,14 @@ def where(self, other, cond, align=True, errors="raise", try_cast=False, axis=0)
 
         # our where function
         def func(cond, values, other):
-            other = self._try_coerce_args(other)
+
+            if not (
+                (self.is_integer or self.is_bool)
+                and lib.is_scalar(other)
+                and np.isnan(other)
+            ):
+                # np.where will cast integer array to floats in this case
+                other = self._try_coerce_args(other)
 
             try:
                 fastres = expressions.where(cond, values, other)