diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py
index 08176af2b326d..5801384bf8db9 100644
--- a/pandas/core/dtypes/cast.py
+++ b/pandas/core/dtypes/cast.py
@@ -350,14 +350,21 @@ def maybe_promote(dtype, fill_value=np.nan):
 
     # returns tuple of (dtype, fill_value)
     if issubclass(dtype.type, np.datetime64):
-        fill_value = tslibs.Timestamp(fill_value).to_datetime64()
+        try:
+            fill_value = tslibs.Timestamp(fill_value).to_datetime64()
+        except (TypeError, ValueError):
+            dtype = np.dtype(np.object_)
     elif issubclass(dtype.type, np.timedelta64):
-        fv = tslibs.Timedelta(fill_value)
-        if fv is NaT:
-            # NaT has no `to_timedelta6` method
-            fill_value = np.timedelta64("NaT", "ns")
+        try:
+            fv = tslibs.Timedelta(fill_value)
+        except ValueError:
+            dtype = np.dtype(np.object_)
         else:
-            fill_value = fv.to_timedelta64()
+            if fv is NaT:
+                # NaT has no `to_timedelta64` method
+                fill_value = np.timedelta64("NaT", "ns")
+            else:
+                fill_value = fv.to_timedelta64()
     elif is_datetime64tz_dtype(dtype):
         if isna(fill_value):
             fill_value = NaT
diff --git a/pandas/tests/dtypes/cast/test_promote.py b/pandas/tests/dtypes/cast/test_promote.py
index cf7a168074e9e..1b7de9b20f42f 100644
--- a/pandas/tests/dtypes/cast/test_promote.py
+++ b/pandas/tests/dtypes/cast/test_promote.py
@@ -272,10 +272,6 @@ def test_maybe_promote_any_with_bool(any_numpy_dtype_reduced, box):
         pytest.xfail("falsely upcasts to object")
     if boxed and dtype not in (str, object) and box_dtype is None:
         pytest.xfail("falsely upcasts to object")
-    if not boxed and dtype.kind == "M":
-        pytest.xfail("raises error")
-    if not boxed and dtype.kind == "m":
-        pytest.xfail("raises error")
 
     # filling anything but bool with bool casts to object
     expected_dtype = np.dtype(object) if dtype != bool else dtype
@@ -348,8 +344,6 @@ def test_maybe_promote_any_with_datetime64(
             or (box_dtype is None and is_datetime64_dtype(type(fill_value)))
         ):
             pytest.xfail("mix of lack of upcasting, resp. wrong missing value")
-        if not boxed and is_timedelta64_dtype(dtype):
-            pytest.xfail("raises error")
 
     # special case for box_dtype
     box_dtype = np.dtype(datetime64_dtype) if box_dtype == "dt_dtype" else box_dtype
@@ -490,9 +484,7 @@ def test_maybe_promote_any_numpy_dtype_with_datetimetz(
     fill_dtype = DatetimeTZDtype(tz=tz_aware_fixture)
     boxed, box_dtype = box  # read from parametrized fixture
 
-    if dtype.kind == "m" and not boxed:
-        pytest.xfail("raises error")
-    elif dtype.kind == "M" and not boxed:
+    if dtype.kind == "M" and not boxed:
         pytest.xfail("Comes back as M8 instead of object")
 
     fill_value = pd.Series([fill_value], dtype=fill_dtype)[0]
@@ -549,8 +541,6 @@ def test_maybe_promote_any_with_timedelta64(
     else:
         if boxed and box_dtype is None and is_timedelta64_dtype(type(fill_value)):
             pytest.xfail("does not upcast correctly")
-        if not boxed and is_datetime64_dtype(dtype):
-            pytest.xfail("raises error")
 
     # special case for box_dtype
     box_dtype = np.dtype(timedelta64_dtype) if box_dtype == "td_dtype" else box_dtype
@@ -622,9 +612,6 @@ def test_maybe_promote_any_with_string(any_numpy_dtype_reduced, string_dtype, bo
     fill_dtype = np.dtype(string_dtype)
     boxed, box_dtype = box  # read from parametrized fixture
 
-    if is_datetime_or_timedelta_dtype(dtype) and box_dtype != object:
-        pytest.xfail("does not upcast or raises")
-
     # create array of given dtype
     fill_value = "abc"
 
@@ -678,9 +665,6 @@ def test_maybe_promote_any_with_object(any_numpy_dtype_reduced, object_dtype, bo
     dtype = np.dtype(any_numpy_dtype_reduced)
     boxed, box_dtype = box  # read from parametrized fixture
 
-    if not boxed and is_datetime_or_timedelta_dtype(dtype):
-        pytest.xfail("raises error")
-
     # create array of object dtype from a scalar value (i.e. passing
     # dtypes.common.is_scalar), which can however not be cast to int/float etc.
     fill_value = pd.DateOffset(1)