Series pct_change fill_method behavior

pandas/core/generic.py

@@ -9953,11 +9953,24 @@ def _check_percentile(self, q):
         """
 
     @Appender(_shared_docs['pct_change'] % _shared_doc_kwargs)
-    def pct_change(self, periods=1, fill_method='pad', limit=None, freq=None,
-                   **kwargs):
+    def pct_change(self, periods=1, fill_method=None, limit=None, freq=None,
+                   skipna=None, **kwargs):
+        if skipna and fill_method is not None:
+            raise ValueError("cannot pass both skipna and fill_method")
+        elif skipna and limit is not None:
+            raise ValueError("cannot pass both skipna and limit")
+        if skipna is None and fill_method is None and limit is None:
+            skipna = True
+        if skipna and self._typ == 'dataframe':
+            return self.apply(
+                lambda s: s.pct_change(periods=periods, freq=freq,
+                                       skipna=skipna, **kwargs)
+            )
         # TODO: Not sure if above is correct - need someone to confirm.
         axis = self._get_axis_number(kwargs.pop('axis', self._stat_axis_name))
-        if fill_method is None:
+        if skipna:
+            data = self.dropna()
+        elif fill_method is None:
             data = self
         else:
             data = self.fillna(method=fill_method, limit=limit, axis=axis)
@@ -9968,6 +9981,8 @@ def pct_change(self, periods=1, fill_method='pad', limit=None, freq=None,
         if freq is None:
             mask = isna(com.values_from_object(data))
             np.putmask(rs.values, mask, np.nan)
+        if skipna:
+            rs = rs.reindex_like(self)
         return rs
 
     def _agg_by_level(self, name, axis=0, level=0, skipna=True, **kwargs):

pandas/tests/frame/test_analytics.py

@@ -1364,6 +1364,20 @@ def test_pct_change(self):
 
             tm.assert_frame_equal(result, expected)
 
+    @pytest.mark.parametrize("periods, expected_vals", [
+        (1, [[np.nan, np.nan], [np.nan, np.nan], [1., np.nan], [0.5, 1.],
+             [np.nan, 0.5], [0.33333333, np.nan], [np.nan, 0.33333333]]),
+        (2, [[np.nan, np.nan], [np.nan, np.nan], [np.nan, np.nan],
+             [2., np.nan], [np.nan, 2.], [1., np.nan], [np.nan, 1.]])
+    ])
+    def test_pct_change_skipna(self, periods, expected_vals):
+        # GH25006
+        df = DataFrame([[np.nan, np.nan], [1., np.nan], [2., 1.], [3., 2.],
+                        [np.nan, 3.], [4., np.nan], [np.nan, 4.]])
+        result = df.pct_change(skipna=True, periods=periods)
+        expected = DataFrame(expected_vals)
+        tm.assert_frame_equal(result, expected)
+
     # ----------------------------------------------------------------------
     # Index of max / min
 

pandas/tests/frame/test_timeseries.py

@@ -143,7 +143,7 @@ def test_pct_change_shift_over_nas(self):
 
         df = DataFrame({'a': s, 'b': s})
 
-        chg = df.pct_change()
+        chg = df.pct_change(fill_method='ffill')
         expected = Series([np.nan, 0.5, 0., 2.5 / 1.5 - 1, .2])
         edf = DataFrame({'a': expected, 'b': expected})
         assert_frame_equal(chg, edf)

pandas/tests/generic/test_generic.py

@@ -607,6 +607,22 @@ def test_pct_change(self, periods, fill_method, limit, exp):
         else:
             tm.assert_series_equal(res, Series(exp))
 
+    @pytest.mark.parametrize('fill_method, limit', [
+        ('backfill', None),
+        ('bfill', None),
+        ('pad', None),
+        ('ffill', None),
+        (None, 1)
+    ])
+    def test_pct_change_skipna_raises(self, fill_method, limit):
+        # GH25006
+        if self._typ is DataFrame or self._typ is Series:
+            vals = [np.nan, np.nan, 1, 2, np.nan, 4, 10, np.nan]
+            obj = self._typ(vals)
+            with pytest.raises(ValueError):
+                obj.pct_change(skipna=True, fill_method=fill_method,
+                               limit=limit)
+
 
 class TestNDFrame(object):
     # tests that don't fit elsewhere

pandas/tests/series/test_analytics.py

@@ -6,7 +6,6 @@
 import operator
 
 import numpy as np
-from numpy import nan
 import pytest
 
 from pandas.compat import PY35, lrange, range
@@ -212,6 +211,18 @@ def test_cummax_timedelta64(self):
         result = s.cummax(skipna=False)
         tm.assert_series_equal(expected, result)
 
+    @pytest.mark.parametrize("periods, expected_vals", [
+        (1, [np.nan, np.nan, 1.0, 0.5, np.nan, 0.333333333333333, np.nan]),
+        (2, [np.nan, np.nan, np.nan, 2.0, np.nan, 1.0, np.nan])
+    ])
+    def test_pct_change_skipna(self, periods, expected_vals):
+        # GH25006
+        vals = [np.nan, 1., 2., 3., np.nan, 4., np.nan]
+        s = Series(vals)
+        result = s.pct_change(skipna=True, periods=periods)
+        expected = Series(expected_vals)
+        assert_series_equal(expected, result)
+
     def test_npdiff(self):
         pytest.skip("skipping due to Series no longer being an "
                     "ndarray")
@@ -220,7 +231,7 @@ def test_npdiff(self):
         s = Series(np.arange(5))
 
         r = np.diff(s)
-        assert_series_equal(Series([nan, 0, 0, 0, nan]), r)
+        assert_series_equal(Series([np.nan, 0, 0, 0, np.nan]), r)
 
     def _check_accum_op(self, name, datetime_series_, check_dtype=True):
         func = getattr(np, name)
@@ -445,14 +456,14 @@ def test_count(self, datetime_series):
 
         assert datetime_series.count() == np.isfinite(datetime_series).sum()
 
-        mi = MultiIndex.from_arrays([list('aabbcc'), [1, 2, 2, nan, 1, 2]])
+        mi = MultiIndex.from_arrays([list('aabbcc'), [1, 2, 2, np.nan, 1, 2]])
         ts = Series(np.arange(len(mi)), index=mi)
 
         left = ts.count(level=1)
-        right = Series([2, 3, 1], index=[1, 2, nan])
+        right = Series([2, 3, 1], index=[1, 2, np.nan])
         assert_series_equal(left, right)
 
-        ts.iloc[[0, 3, 5]] = nan
+        ts.iloc[[0, 3, 5]] = np.nan
         assert_series_equal(ts.count(level=1), right - 1)
 
     def test_dot(self):
@@ -673,11 +684,11 @@ def test_cummethods_bool(self):
             result = getattr(s, method)()
             assert_series_equal(result, expected)
 
-        e = pd.Series([False, True, nan, False])
-        cse = pd.Series([0, 1, nan, 1], dtype=object)
-        cpe = pd.Series([False, 0, nan, 0])
-        cmin = pd.Series([False, False, nan, False])
-        cmax = pd.Series([False, True, nan, True])
+        e = pd.Series([False, True, np.nan, False])
+        cse = pd.Series([0, 1, np.nan, 1], dtype=object)
+        cpe = pd.Series([False, 0, np.nan, 0])
+        cmin = pd.Series([False, False, np.nan, False])
+        cmax = pd.Series([False, True, np.nan, True])
         expecteds = {'cumsum': cse,
                      'cumprod': cpe,
                      'cummin': cmin,
@@ -954,15 +965,13 @@ def test_shift_categorical(self):
         assert_index_equal(s.values.categories, sn2.values.categories)
 
     def test_unstack(self):
-        from numpy import nan
-
         index = MultiIndex(levels=[['bar', 'foo'], ['one', 'three', 'two']],
                            codes=[[1, 1, 0, 0], [0, 1, 0, 2]])
 
         s = Series(np.arange(4.), index=index)
         unstacked = s.unstack()
 
-        expected = DataFrame([[2., nan, 3.], [0., 1., nan]],
+        expected = DataFrame([[2., np.nan, 3.], [0., 1., np.nan]],
                              index=['bar', 'foo'],
                              columns=['one', 'three', 'two'])
 
@@ -986,17 +995,17 @@ def test_unstack(self):
         idx = pd.MultiIndex.from_arrays([[101, 102], [3.5, np.nan]])
         ts = pd.Series([1, 2], index=idx)
         left = ts.unstack()
-        right = DataFrame([[nan, 1], [2, nan]], index=[101, 102],
-                          columns=[nan, 3.5])
+        right = DataFrame([[np.nan, 1], [2, np.nan]], index=[101, 102],
+                          columns=[np.nan, 3.5])
         assert_frame_equal(left, right)
 
         idx = pd.MultiIndex.from_arrays([['cat', 'cat', 'cat', 'dog', 'dog'
                                           ], ['a', 'a', 'b', 'a', 'b'],
                                          [1, 2, 1, 1, np.nan]])
         ts = pd.Series([1.0, 1.1, 1.2, 1.3, 1.4], index=idx)
-        right = DataFrame([[1.0, 1.3], [1.1, nan], [nan, 1.4], [1.2, nan]],
-                          columns=['cat', 'dog'])
-        tpls = [('a', 1), ('a', 2), ('b', nan), ('b', 1)]
+        right = DataFrame([[1.0, 1.3], [1.1, np.nan], [np.nan, 1.4],
+                           [1.2, np.nan]], columns=['cat', 'dog'])
+        tpls = [('a', 1), ('a', 2), ('b', np.nan), ('b', 1)]
         right.index = pd.MultiIndex.from_tuples(tpls)
         assert_frame_equal(ts.unstack(level=0), right)
 

pandas/tests/series/test_timeseries.py

@@ -399,7 +399,7 @@ def test_pct_change(self):
     def test_pct_change_shift_over_nas(self):
         s = Series([1., 1.5, np.nan, 2.5, 3.])
 
-        chg = s.pct_change()
+        chg = s.pct_change(fill_method='ffill')
         expected = Series([np.nan, 0.5, 0., 2.5 / 1.5 - 1, .2])
         assert_series_equal(chg, expected)