ENH: fill_value argument for shift

Author: ahcub

pandas/core/arrays/datetimelike.py

@@ -780,7 +780,7 @@ def _addsub_offset_array(self, other, op):
         return self._from_sequence(res_values)
 
     @deprecate_kwarg(old_arg_name='n', new_arg_name='periods')
-    def shift(self, periods, freq=None):
+    def shift(self, periods, freq=None, fill_value=np.nan):
         """
         Shift index by desired number of time frequency increments.
 
@@ -810,9 +810,9 @@ def shift(self, periods, freq=None):
         Index.shift : Shift values of Index.
         PeriodIndex.shift : Shift values of PeriodIndex.
         """
-        return self._time_shift(periods=periods, freq=freq)
+        return self._time_shift(periods=periods, freq=freq, fill_value=np.nan)
 
-    def _time_shift(self, periods, freq=None):
+    def _time_shift(self, periods, freq=None, fill_value=np.nan):
         """
         Shift each value by `periods`.
 

pandas/core/frame.py

@@ -3867,9 +3867,9 @@ def replace(self, to_replace=None, value=None, inplace=False, limit=None,
                                               method=method)
 
     @Appender(_shared_docs['shift'] % _shared_doc_kwargs)
-    def shift(self, periods=1, freq=None, axis=0):
+    def shift(self, periods=1, freq=None, axis=0, fill_value=np.nan):
         return super(DataFrame, self).shift(periods=periods, freq=freq,
-                                            axis=axis)
+                                            axis=axis, fill_value=fill_value)
 
     def set_index(self, keys, drop=True, append=False, inplace=False,
                   verify_integrity=False):

pandas/core/generic.py

@@ -8868,15 +8868,15 @@ def mask(self, cond, other=np.nan, inplace=False, axis=None, level=None,
     """)
 
     @Appender(_shared_docs['shift'] % _shared_doc_kwargs)
-    def shift(self, periods=1, freq=None, axis=0):
+    def shift(self, periods=1, freq=None, axis=0, fill_value=np.nan):
         if periods == 0:
             return self.copy()
 
         block_axis = self._get_block_manager_axis(axis)
         if freq is None:
-            new_data = self._data.shift(periods=periods, axis=block_axis)
+            new_data = self._data.shift(periods=periods, axis=block_axis, fill_value=fill_value)
         else:
-            return self.tshift(periods, freq)
+            return self.tshift(periods, freq, fill_value=fill_value)
 
         return self._constructor(new_data).__finalize__(self)
 
@@ -8916,7 +8916,7 @@ def slice_shift(self, periods=1, axis=0):
 
         return new_obj.__finalize__(self)
 
-    def tshift(self, periods=1, freq=None, axis=0):
+    def tshift(self, periods=1, freq=None, axis=0, fill_value=np.nan):
         """
         Shift the time index, using the index's frequency if available.
 

pandas/core/internals/blocks.py

@@ -1260,12 +1260,12 @@ def diff(self, n, axis=1):
         new_values = algos.diff(self.values, n, axis=axis)
         return [self.make_block(values=new_values)]
 
-    def shift(self, periods, axis=0):
+    def shift(self, periods, axis=0, fill_value=np.nan):
         """ shift the block by periods, possibly upcast """
 
         # convert integer to float if necessary. need to do a lot more than
         # that, handle boolean etc also
-        new_values, fill_value = maybe_upcast(self.values)
+        new_values, fill_value = maybe_upcast(self.values, fill_value)
 
         # make sure array sent to np.roll is c_contiguous
         f_ordered = new_values.flags.f_contiguous

pandas/core/series.py

@@ -3714,8 +3714,8 @@ def replace(self, to_replace=None, value=None, inplace=False, limit=None,
                                            regex=regex, method=method)
 
     @Appender(generic._shared_docs['shift'] % _shared_doc_kwargs)
-    def shift(self, periods=1, freq=None, axis=0):
-        return super(Series, self).shift(periods=periods, freq=freq, axis=axis)
+    def shift(self, periods=1, freq=None, axis=0, fill_value=np.nan):
+        return super(Series, self).shift(periods=periods, freq=freq, axis=axis, fill_value=fill_value)
 
     def reindex_axis(self, labels, axis=0, **kwargs):
         """

pandas/tests/series/test_timeseries.py

@@ -129,6 +129,37 @@ def test_shift2(self):
         idx = DatetimeIndex(['2000-01-01', '2000-01-02', '2000-01-04'])
         pytest.raises(NullFrequencyError, idx.shift, 1)
 
+    def test_shift_fill_value(self):
+        ts = Series(np.random.randn(5),
+                    index=date_range('1/1/2000', periods=5, freq='H'))
+
+        # fill_value should have no effect on shift with freq
+        result = ts.shift(1, freq='5T', fill_value=0)
+        exp_index = ts.index.shift(1, freq='5T')
+        tm.assert_index_equal(result.index, exp_index)
+        tm.assert_equal(result.iloc[0].value, ts.iloc[0].value)
+
+        # check that fill value works
+        result = ts.shift(1, fill_value=0.0)
+        tm.assert_equal(result.iloc[0].value, 0.0)
+        tm.assert_equal(result.iloc[1].value, ts.iloc[0].value)
+
+        result = ts.shift(2, fill_value=0.0)
+        tm.assert_equal(result.iloc[0].value, 0.0)
+        tm.assert_equal(result.iloc[1].value, 0.0)
+        tm.assert_equal(result.iloc[2].value, ts.iloc[0].value)
+
+        df = DataFrame(np.random.randn(5), index=date_range('1/1/2000', periods=5, freq='H'))
+
+        result = df.shift(1, fill_value=0.0)
+        tm.assert_equal(result.iloc[0, 0].value, 0.0)
+        tm.assert_equal(result.iloc[1, 0].value, df.iloc[0, 0].value)
+
+        result = df.shift(2, fill_value=0.0)
+        tm.assert_equal(result.iloc[0, 0].value, 0.0)
+        tm.assert_equal(result.iloc[1, 0].value, 0.0)
+        tm.assert_equal(result.iloc[2, 0].value, df.iloc[0, 0].value)
+
     def test_shift_dst(self):
         # GH 13926
         dates = date_range('2016-11-06', freq='H', periods=10, tz='US/Eastern')