diff --git a/doc/source/release.rst b/doc/source/release.rst
index 78c92ec11609e..7109b87f5352b 100644
--- a/doc/source/release.rst
+++ b/doc/source/release.rst
@@ -64,6 +64,8 @@ Experimental Features
 Improvements to existing features
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
+  - perf improvements in Series datetime/timedelta binary operations (:issue:`5801`)
+
 Bug Fixes
 ~~~~~~~~~
 
diff --git a/pandas/src/inference.pyx b/pandas/src/inference.pyx
index dce46c972fb3b..84f1f3cb4904d 100644
--- a/pandas/src/inference.pyx
+++ b/pandas/src/inference.pyx
@@ -35,6 +35,8 @@ def infer_dtype(object _values):
 
     if isinstance(_values, np.ndarray):
         values = _values
+    elif hasattr(_values,'values'):
+        values = _values.values
     else:
         if not isinstance(_values, list):
             _values = list(_values)
diff --git a/vb_suite/binary_ops.py b/vb_suite/binary_ops.py
index fc84dd8bcdb81..5ec2d9fcfc2cf 100644
--- a/vb_suite/binary_ops.py
+++ b/vb_suite/binary_ops.py
@@ -103,6 +103,9 @@
     Benchmark("df[(df>0) & (df2>0)]", setup, name='frame_multi_and_no_ne',cleanup="expr.set_use_numexpr(True)",
               start_date=datetime(2013, 2, 26))
 
+#----------------------------------------------------------------------
+# timeseries
+
 setup = common_setup + """
 N = 1000000
 halfway = N // 2 - 1
@@ -114,3 +117,13 @@
                                      start_date=datetime(2013, 9, 27))
 series_timestamp_compare = Benchmark("s <= ts", setup,
                                      start_date=datetime(2012, 2, 21))
+
+setup = common_setup + """
+N = 1000000
+s = Series(date_range('20010101', periods=N, freq='s'))
+"""
+
+timestamp_ops_diff1 = Benchmark("s.diff()", setup,
+                                start_date=datetime(2013, 1, 1))
+timestamp_ops_diff2 = Benchmark("s-s.shift()", setup,
+                                start_date=datetime(2013, 1, 1))