diff --git a/doc/source/release.rst b/doc/source/release.rst
index 578e235b0f88b..dbfd5bf717cbe 100644
--- a/doc/source/release.rst
+++ b/doc/source/release.rst
@@ -61,6 +61,8 @@ pandas 0.13
     is frequency conversion.
   - Performance improvements with ``__getitem__`` on ``DataFrames`` with
     when the key is a column
+  - Support for using a ``DatetimeIndex/PeriodsIndex`` directly in a datelike calculation
+    e.g. s-s.index (:issue:`4629`)
 
 **API Changes**
 
diff --git a/pandas/core/series.py b/pandas/core/series.py
index 0fc3341c52117..050a9de2b23dc 100644
--- a/pandas/core/series.py
+++ b/pandas/core/series.py
@@ -103,7 +103,7 @@ def wrapper(self, other, name=name):
             coerce = 'compat' if _np_version_under1p7 else True
 
             # convert the argument to an ndarray
-            def convert_to_array(values):
+            def convert_to_array(values, other=None):
                 if not is_list_like(values):
                     values = np.array([values])
                 inferred_type = lib.infer_dtype(values)
@@ -111,6 +111,8 @@ def convert_to_array(values):
                     # a datetlike
                     if not (isinstance(values, (pa.Array, Series)) and com.is_datetime64_dtype(values)):
                         values = tslib.array_to_datetime(values)
+                    elif isinstance(values, DatetimeIndex):
+                        other = values = values.to_series()
                 elif inferred_type in set(['timedelta']):
                     # have a timedelta, convert to to ns here
                     values = com._possibly_cast_to_timedelta(values, coerce=coerce)
@@ -121,6 +123,8 @@ def convert_to_array(values):
                     # py3 compat where dtype is 'm' but is an integer
                     if values.dtype.kind == 'm':
                         values = values.astype('timedelta64[ns]')
+                    elif isinstance(values, PeriodIndex):
+                        other = values = values.to_timestamp().to_series()
                     elif name not in ['__truediv__','__div__','__mul__']:
                         raise TypeError("incompatible type for a datetime/timedelta operation [{0}]".format(name))
                 elif isinstance(values[0],DateOffset):
@@ -134,11 +138,11 @@ def convert_to_array(values):
                 else:
                     raise TypeError("incompatible type [{0}] for a datetime/timedelta operation".format(pa.array(values).dtype))
 
-                return values
+                return values, other
 
             # convert lhs and rhs
-            lvalues = convert_to_array(lvalues)
-            rvalues = convert_to_array(rvalues)
+            lvalues,_ = convert_to_array(lvalues)
+            rvalues,other = convert_to_array(rvalues,other)
 
             is_datetime_rhs  = com.is_datetime64_dtype(rvalues)
             is_timedelta_rhs = com.is_timedelta64_dtype(rvalues) or (not is_datetime_rhs and _np_version_under1p7)
diff --git a/pandas/tests/test_series.py b/pandas/tests/test_series.py
index 4100185752b8f..a6e7d7cbc1773 100644
--- a/pandas/tests/test_series.py
+++ b/pandas/tests/test_series.py
@@ -2319,6 +2319,30 @@ def test_sub_of_datetime_from_TimeSeries(self):
         result = com._possibly_cast_to_timedelta(np.abs(a - b))
         self.assert_(result.dtype == 'timedelta64[ns]')
 
+    def test_datetime64_with_index(self):
+
+        # arithmetic integer ops with an index
+        s = Series(np.random.randn(5))
+        expected = s-s.index.to_series()
+        result = s-s.index
+        assert_series_equal(result,expected)
+
+        # GH 4629
+        # arithmetic datetime64 ops with an index
+        s = Series(date_range('20130101',periods=5),index=date_range('20130101',periods=5))
+        expected = s-s.index.to_series()
+        result = s-s.index
+        assert_series_equal(result,expected)
+
+        result = s-s.index.to_period()
+        assert_series_equal(result,expected)
+
+        df = DataFrame(np.random.randn(5,2),index=date_range('20130101',periods=5))
+        df['date'] = Timestamp('20130102')
+        df['expected'] = df['date']-df.index.to_series()
+        df['result'] = df['date']-df.index
+        assert_series_equal(df['result'],df['expected'])
+
     def test_timedelta64_nan(self):
 
         from pandas import tslib