diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py
index 4435b2518e90b..b439a3e2dfbc8 100644
--- a/pandas/core/dtypes/cast.py
+++ b/pandas/core/dtypes/cast.py
@@ -359,6 +359,8 @@ def maybe_promote(dtype, fill_value=np.nan):
         if isinstance(fill_value, datetime) and fill_value.tzinfo is not None:
             # Trying to insert tzaware into tznaive, have to cast to object
             dtype = np.dtype(np.object_)
+        elif is_integer(fill_value) or (is_float(fill_value) and not isna(fill_value)):
+            dtype = np.dtype(np.object_)
         else:
             try:
                 fill_value = tslibs.Timestamp(fill_value).to_datetime64()
diff --git a/pandas/tests/dtypes/cast/test_promote.py b/pandas/tests/dtypes/cast/test_promote.py
index da2b4c28a02a5..2252e6c7b3dc9 100644
--- a/pandas/tests/dtypes/cast/test_promote.py
+++ b/pandas/tests/dtypes/cast/test_promote.py
@@ -603,8 +603,6 @@ def test_maybe_promote_datetime64_with_any(
     else:
         if boxed and box_dtype is None:
             pytest.xfail("does not upcast to object")
-        if not boxed:
-            pytest.xfail("does not upcast to object or raises")
 
     # create array of given dtype; casts "1" to correct dtype
     fill_value = np.array([1], dtype=fill_dtype)[0]