FIX: fix interpolate with kwarg limit area and limit direction using pad or bfill

doc/source/whatsnew/v1.0.0.rst

@@ -1040,7 +1040,7 @@ Indexing
 Missing
 ^^^^^^^
 
--
+-  Bug in :meth:`Series.interpolate` where kwarg ``limit_area`` and ``limit_direction`` had no effect when using methods ``pad`` and `backfill`` (:issue:`31048`)
 -
 
 MultiIndex

pandas/core/generic.py

@@ -6527,9 +6527,11 @@ def replace(
             0.
         inplace : bool, default False
             Update the data in place if possible.
-        limit_direction : {'forward', 'backward', 'both'}, default 'forward'
+        limit_direction : {'forward', 'backward', 'both'}, default is None
             If limit is specified, consecutive NaNs will be filled in this
-            direction.
+            direction. If the methods 'pad' or 'ffill' are used it must be
+            None or 'forward'. If 'backfill' or 'bfill' are use it must be
+            None or 'backwards'.
         limit_area : {`None`, 'inside', 'outside'}, default None
             If limit is specified, consecutive NaNs will be filled with this
             restriction.
@@ -6680,7 +6682,7 @@ def interpolate(
         axis=0,
         limit=None,
         inplace=False,
-        limit_direction="forward",
+        limit_direction=None,
         limit_area=None,
         downcast=None,
         **kwargs,
@@ -6720,6 +6722,28 @@ def interpolate(
                 "column to a numeric dtype."
             )
 
+        # Set `limit_direction` depending on `method`
+        if (method == "pad") or (method == "ffill"):
+            if (limit_direction == "backward") or (limit_direction == "both"):
+                raise ValueError(
+                    f"`limit_direction` must not be `{limit_direction}` "
+                    f"for method `{method}`"
+                )
+            else:
+                limit_direction = "forward"
+        elif (method == "backfill") or (method == "bfill"):
+            if (limit_direction == "forward") or (limit_direction == "both"):
+                raise ValueError(
+                    f"`limit_direction` must not be `{limit_direction}` "
+                    f"for method `{method}`"
+                )
+            else:
+                limit_direction = "backward"
+        else:
+            # Set default
+            if limit_direction is None:
+                limit_direction = "forward"
+
         # create/use the index
         if method == "linear":
             # prior default

pandas/core/internals/blocks.py

@@ -1116,6 +1116,7 @@ def check_int_bool(self, inplace):
                 axis=axis,
                 inplace=inplace,
                 limit=limit,
+                limit_area=limit_area,
                 fill_value=fill_value,
                 coerce=coerce,
                 downcast=downcast,
@@ -1146,6 +1147,7 @@ def _interpolate_with_fill(
         axis=0,
         inplace=False,
         limit=None,
+        limit_area=None,
         fill_value=None,
         coerce=False,
         downcast=None,
@@ -1168,16 +1170,36 @@ def _interpolate_with_fill(
         # We only get here for non-ExtensionBlock
         fill_value = convert_scalar(self.values, fill_value)
 
-        values = missing.interpolate_2d(
-            values,
-            method=method,
-            axis=axis,
-            limit=limit,
-            fill_value=fill_value,
-            dtype=self.dtype,
-        )
+        # We have to distinguish two cases:
+        # 1. When kwarg `limit_area` is used: It is not
+        #    supported by `missing.interpolate_2d()`. Using this kwarg only
+        #    works by applying the fill along a certain axis.
+        # 2. All other cases: Then, `missing.interpolate_2d()` can be used.
+        if limit_area is not None:
+
+            def func(x):
+                return missing.interpolate_1d_fill(
+                    x,
+                    method=method,
+                    limit=limit,
+                    limit_area=limit_area,
+                    fill_value=fill_value,
+                    dtype=self.dtype,
+                )
+
+            # Beware that this also changes the input array `values`!
+            interp_values = np.apply_along_axis(func, axis, values)
+        else:
+            interp_values = missing.interpolate_2d(
+                values,
+                method=method,
+                axis=axis,
+                limit=limit,
+                fill_value=fill_value,
+                dtype=self.dtype,
+            )
 
-        blocks = [self.make_block_same_class(values, ndim=self.ndim)]
+        blocks = [self.make_block_same_class(interp_values, ndim=self.ndim)]
         return self._maybe_downcast(blocks, downcast)
 
     def _interpolate(

pandas/core/missing.py

@@ -5,6 +5,7 @@
 import numpy as np
 
 from pandas._libs import algos, lib
+from pandas._typing import Dtype, Hashable, Optional
 from pandas.compat._optional import import_optional_dependency
 
 from pandas.core.dtypes.cast import infer_dtype_from_array
@@ -222,40 +223,14 @@ def interpolate_1d(
     # default limit is unlimited GH #16282
     limit = algos._validate_limit(nobs=None, limit=limit)
 
-    # These are sets of index pointers to invalid values... i.e. {0, 1, etc...
-    all_nans = set(np.flatnonzero(invalid))
-    start_nans = set(range(find_valid_index(yvalues, "first")))
-    end_nans = set(range(1 + find_valid_index(yvalues, "last"), len(valid)))
-    mid_nans = all_nans - start_nans - end_nans
-
-    # Like the sets above, preserve_nans contains indices of invalid values,
-    # but in this case, it is the final set of indices that need to be
-    # preserved as NaN after the interpolation.
-
-    # For example if limit_direction='forward' then preserve_nans will
-    # contain indices of NaNs at the beginning of the series, and NaNs that
-    # are more than'limit' away from the prior non-NaN.
-
-    # set preserve_nans based on direction using _interp_limit
-    if limit_direction == "forward":
-        preserve_nans = start_nans | set(_interp_limit(invalid, limit, 0))
-    elif limit_direction == "backward":
-        preserve_nans = end_nans | set(_interp_limit(invalid, 0, limit))
-    else:
-        # both directions... just use _interp_limit
-        preserve_nans = set(_interp_limit(invalid, limit, limit))
-
-    # if limit_area is set, add either mid or outside indices
-    # to preserve_nans GH #16284
-    if limit_area == "inside":
-        # preserve NaNs on the outside
-        preserve_nans |= start_nans | end_nans
-    elif limit_area == "outside":
-        # preserve NaNs on the inside
-        preserve_nans |= mid_nans
-
-    # sort preserve_nans and covert to list
-    preserve_nans = sorted(preserve_nans)
+    preserve_nans = _derive_indices_of_nans_to_preserve(
+        yvalues=yvalues,
+        valid=valid,
+        invalid=invalid,
+        limit=limit,
+        limit_area=limit_area,
+        limit_direction=limit_direction,
+    )
 
     xvalues = getattr(xvalues, "values", xvalues)
     yvalues = getattr(yvalues, "values", yvalues)
@@ -314,6 +289,53 @@ def interpolate_1d(
         return result
 
 
+def _derive_indices_of_nans_to_preserve(
+    yvalues, valid, invalid, limit, limit_area, limit_direction
+):
+    """ Derive the indices of NaNs that shall be preserved after interpolation
+    This function is called by `interpolate_1d` and takes the arguments with
+    the same name from there. In `interpolate_1d`, after performing the
+    interpolation the list of indices of NaNs to preserve is used to put
+    NaNs in the desired locations.
+    """
+
+    # These are sets of index pointers to invalid values... i.e. {0, 1, etc...
+    all_nans = set(np.flatnonzero(invalid))
+    start_nans = set(range(find_valid_index(yvalues, "first")))
+    end_nans = set(range(1 + find_valid_index(yvalues, "last"), len(valid)))
+    mid_nans = all_nans - start_nans - end_nans
+
+    # Like the sets above, preserve_nans contains indices of invalid values,
+    # but in this case, it is the final set of indices that need to be
+    # preserved as NaN after the interpolation.
+
+    # For example if limit_direction='forward' then preserve_nans will
+    # contain indices of NaNs at the beginning of the series, and NaNs that
+    # are more than'limit' away from the prior non-NaN.
+
+    # set preserve_nans based on direction using _interp_limit
+    if limit_direction == "forward":
+        preserve_nans = start_nans | set(_interp_limit(invalid, limit, 0))
+    elif limit_direction == "backward":
+        preserve_nans = end_nans | set(_interp_limit(invalid, 0, limit))
+    else:
+        # both directions... just use _interp_limit
+        preserve_nans = set(_interp_limit(invalid, limit, limit))
+
+    # if limit_area is set, add either mid or outside indices
+    # to preserve_nans GH #16284
+    if limit_area == "inside":
+        # preserve NaNs on the outside
+        preserve_nans |= start_nans | end_nans
+    elif limit_area == "outside":
+        # preserve NaNs on the inside
+        preserve_nans |= mid_nans
+
+    # sort preserve_nans and covert to list
+    preserve_nans = sorted(preserve_nans)
+    return preserve_nans
+
+
 def _interpolate_scipy_wrapper(
     x, y, new_x, method, fill_value=None, bounds_error=False, order=None, **kwargs
 ):
@@ -478,6 +500,66 @@ def _akima_interpolate(xi, yi, x, der=0, axis=0):
         return [P(x, nu) for nu in der]
 
 
+def interpolate_1d_fill(
+    values,
+    method: str = "pad",
+    limit: Optional[int] = None,
+    limit_area: Optional[str] = None,
+    fill_value: Optional[Hashable] = None,
+    dtype: Optional[Dtype] = None,
+):
+    """
+    This is a  1D-versoin of `interpolate_2d`, which is used for methods `pad`
+    and `backfill` when interpolating. This 1D-version is necessary to be
+    able to handle kwarg `limit_area` via the function
+    ` _derive_indices_of_nans_to_preserve`. It is used the same way as the
+    1D-interpolation functions which are based on scipy-interpolation, i.e.
+    via np.apply_along_axis.
+    """
+    if method == "pad":
+        limit_direction = "forward"
+    elif method == "backfill":
+        limit_direction = "backward"
+    else:
+        raise ValueError("`method` must be either 'pad' or 'backfill'.")
+
+    orig_values = values
+
+    yvalues = values
+    invalid = isna(yvalues)
+    valid = ~invalid
+
+    if values.ndim > 1:
+        raise AssertionError("This only works with 1D data.")
+
+    if fill_value is None:
+        mask = None
+    else:  # todo create faster fill func without masking
+        mask = mask_missing(values, fill_value)
+
+    preserve_nans = _derive_indices_of_nans_to_preserve(
+        yvalues=yvalues,
+        valid=valid,
+        invalid=invalid,
+        limit=limit,
+        limit_area=limit_area,
+        limit_direction=limit_direction,
+    )
+
+    method = clean_fill_method(method)
+    if method == "pad":
+        values = pad_1d(values, limit=limit, mask=mask, dtype=dtype)
+    else:
+        values = backfill_1d(values, limit=limit, mask=mask, dtype=dtype)
+
+    if orig_values.dtype.kind == "M":
+        # convert float back to datetime64
+        values = values.astype(orig_values.dtype)
+
+    values[preserve_nans] = fill_value
+    return values
+
+
 def interpolate_2d(
     values, method="pad", axis=0, limit=None, fill_value=None, dtype=None
 ):

pandas/tests/series/test_missing.py

@@ -1382,6 +1382,75 @@ def test_interp_limit_area(self):
         with pytest.raises(ValueError, match=msg):
             s.interpolate(method="linear", limit_area="abc")
 
+    def test_interp_limit_area_with_pad(self):
+        # Test for issue #26796
+        s = Series([np.nan, np.nan, 3, np.nan, np.nan, np.nan, 7, np.nan, np.nan])
+
+        expected = Series([np.nan, np.nan, 3.0, 3.0, 3.0, 3.0, 7.0, np.nan, np.nan])
+        result = s.interpolate(method="pad", limit_area="inside")
+        tm.assert_series_equal(result, expected)
+
+        expected = Series(
+            [np.nan, np.nan, 3.0, 3.0, np.nan, np.nan, 7.0, np.nan, np.nan]
+        )
+        result = s.interpolate(method="pad", limit_area="inside", limit=1)
+        tm.assert_series_equal(result, expected)
+
+        expected = Series([np.nan, np.nan, 3.0, np.nan, np.nan, np.nan, 7.0, 7.0, 7.0])
+        result = s.interpolate(method="pad", limit_area="outside")
+        tm.assert_series_equal(result, expected)
+
+        expected = Series(
+            [np.nan, np.nan, 3.0, np.nan, np.nan, np.nan, 7.0, 7.0, np.nan]
+        )
+        result = s.interpolate(method="pad", limit_area="outside", limit=1)
+        tm.assert_series_equal(result, expected)
+
+    def test_interp_limit_area_with_backfill(self):
+        # Test for issue #26796
+        s = Series([np.nan, np.nan, 3, np.nan, np.nan, np.nan, 7, np.nan, np.nan])
+
+        expected = Series([np.nan, np.nan, 3.0, 7.0, 7.0, 7.0, 7.0, np.nan, np.nan])
+        result = s.interpolate(method="bfill", limit_area="inside")
+        tm.assert_series_equal(result, expected)
+
+        expected = Series(
+            [np.nan, np.nan, 3.0, np.nan, np.nan, 7.0, 7.0, np.nan, np.nan]
+        )
+        result = s.interpolate(method="bfill", limit_area="inside", limit=1)
+        tm.assert_series_equal(result, expected)
+
+        expected = Series([3.0, 3.0, 3.0, np.nan, np.nan, np.nan, 7.0, np.nan, np.nan])
+        result = s.interpolate(method="bfill", limit_area="outside")
+        tm.assert_series_equal(result, expected)
+
+        expected = Series(
+            [np.nan, 3.0, 3.0, np.nan, np.nan, np.nan, 7.0, np.nan, np.nan]
+        )
+        result = s.interpolate(method="bfill", limit_area="outside", limit=1)
+        tm.assert_series_equal(result, expected)
+
+    def test_interp_raise_limit_direction_and_pad_or_bfill(self):
+        s = Series([1, 2, 3])
+        forbidden_combinations = [
+            ("pad", "backward"),
+            ("ffill", "backward"),
+            ("backfill", "forward"),
+            ("bfill", "forward"),
+            ("pad", "both"),
+            ("ffill", "both"),
+            ("backfill", "both"),
+            ("bfill", "both"),
+        ]
+
+        for method, limit_direction in forbidden_combinations:
+            msg = (
+                f"`limit_direction` must not be `{limit_direction}` "
+                f"for method `{method}`"
+            )
+            with pytest.raises(ValueError, match=msg):
+                s.interpolate(method=method, limit_direction=limit_direction)
+
     def test_interp_limit_direction(self):
         # These tests are for issue #9218 -- fill NaNs in both directions.
         s = Series([1, 3, np.nan, np.nan, np.nan, 11])