diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py
index e550309461de4..925a918910703 100644
--- a/pandas/core/dtypes/cast.py
+++ b/pandas/core/dtypes/cast.py
@@ -491,37 +491,23 @@ def maybe_casted_values(
     if codes is not None:
         mask: np.ndarray = codes == -1
 
-        # we can have situations where the whole mask is -1,
-        # meaning there is nothing found in codes, so make all nan's
         if mask.size > 0 and mask.all():
+            # we can have situations where the whole mask is -1,
+            # meaning there is nothing found in codes, so make all nan's
+
             dtype = index.dtype
             fill_value = na_value_for_dtype(dtype)
             values = construct_1d_arraylike_from_scalar(fill_value, len(mask), dtype)
+
         else:
             values = values.take(codes)
 
-            # TODO(https://github.com/pandas-dev/pandas/issues/24206)
-            # Push this into maybe_upcast_putmask?
-            # We can't pass EAs there right now. Looks a bit
-            # complicated.
-            # So we unbox the ndarray_values, op, re-box.
-            values_type = type(values)
-            values_dtype = values.dtype
-
-            from pandas.core.arrays.datetimelike import DatetimeLikeArrayMixin
-
-            if isinstance(values, DatetimeLikeArrayMixin):
-                values = values._data  # TODO: can we de-kludge yet?
-
             if mask.any():
                 if isinstance(values, np.ndarray):
                     values, _ = maybe_upcast_putmask(values, mask, np.nan)
                 else:
                     values[mask] = np.nan
 
-            if issubclass(values_type, DatetimeLikeArrayMixin):
-                values = values_type(values, dtype=values_dtype)
-
     return values