diff --git a/pandas/core/arrays/arrow/array.py b/pandas/core/arrays/arrow/array.py
index 3c65e6b4879e2..43320cf68cbec 100644
--- a/pandas/core/arrays/arrow/array.py
+++ b/pandas/core/arrays/arrow/array.py
@@ -952,6 +952,9 @@ def convert_fill_value(value, pa_type, dtype):
                 return value
             if isinstance(value, (pa.Scalar, pa.Array, pa.ChunkedArray)):
                 return value
+            if isinstance(value, Timedelta) and value.unit in ("s", "ms"):
+                # Workaround https://github.com/apache/arrow/issues/37291
+                value = value.to_numpy()
             if is_array_like(value):
                 pa_box = pa.array
             else: