fix: Fix time series interpolation on resampled data for off-grid data points

Fixes pandas-dev#21351 (Time Series Interpolation is wrong). Interpolation on resampled data was only working as the original data points were discarded in the resampling process if they were not aligned with the resampled index. In this implementation the missing datapoints are added to the dataset before interpolation, and discarded afterwards. Does not handle PeriodIndex.

Author: cbpygit

pandas/core/missing.py

@@ -304,7 +304,16 @@ def get_interp_index(method, index: Index) -> Index:
         # prior default
         from pandas import Index
 
-        index = Index(np.arange(len(index)))
+        if isinstance(index.dtype, DatetimeTZDtype) or lib.is_np_dtype(
+            index.dtype, "mM"
+        ):
+            # Convert datetime-like indexes to int64
+            index = Index(index.view("i8"))
+
+        elif not is_numeric_dtype(index.dtype):
+            # We keep behavior consistent with prior versions of pandas for
+            # non-numeric, non-datetime indexes
+            index = Index(np.arange(len(index)))
     else:
         methods = {"index", "values", "nearest", "time"}
         is_numeric_or_datetime = (

pandas/core/resample.py

@@ -83,6 +83,7 @@
     TimedeltaIndex,
     timedelta_range,
 )
+from pandas.core.reshape.concat import concat
 
 from pandas.tseries.frequencies import (
     is_subperiod,
@@ -1085,7 +1086,23 @@ def interpolate(
         """
         assert downcast is lib.no_default  # just checking coverage
         result = self._upsample("asfreq")
-        return result.interpolate(
+
+        # If the original data has timestamps which are not aligned with the
+        # target timestamps, we need to add those points back to the data frame
+        # that is supposed to be interpolated. This does not work with
+        # PeriodIndex, so we skip this case.
+        obj = self._selected_obj
+        is_period_index = isinstance(obj.index, PeriodIndex)
+
+        if not is_period_index:
+            final_index = result.index
+            missing_data_points_index = obj.index.difference(final_index)
+            if len(missing_data_points_index) > 0:
+                result = concat(
+                    [result, obj.loc[missing_data_points_index]]
+                ).sort_index()
+
+        result_interpolated = result.interpolate(
             method=method,
             axis=axis,
             limit=limit,
@@ -1096,6 +1113,12 @@ def interpolate(
             **kwargs,
         )
 
+        # We make sure that original data points which do not align with the
+        # resampled index are removed
+        if is_period_index:
+            return result_interpolated
+        return result_interpolated.loc[final_index]
+
     @final
     def asfreq(self, fill_value=None):
         """

pandas/tests/resample/test_base.py

@@ -36,6 +36,27 @@
     [DATE_RANGE, PERIOD_RANGE, TIMEDELTA_RANGE],
 )
 
+all_1d_no_arg_interpolation_methods = pytest.mark.parametrize(
+    "method",
+    [
+        "linear",
+        "time",
+        "index",
+        "values",
+        "nearest",
+        "zero",
+        "slinear",
+        "quadratic",
+        "cubic",
+        "barycentric",
+        "krogh",
+        "from_derivatives",
+        "piecewise_polynomial",
+        "pchip",
+        "akima",
+    ],
+)
+
 
 @pytest.fixture
 def create_index(_index_factory):
@@ -90,6 +111,53 @@ def test_resample_interpolate(frame):
     tm.assert_frame_equal(result, expected)
 
 
+@all_1d_no_arg_interpolation_methods
+def test_resample_interpolate_regular_sampling_off_grid(method):
+    # GH#21351
+    index = date_range("2000-01-01 00:01:00", periods=5, freq="2h")
+    ser = Series(np.arange(5.0), index)
+
+    # Resample to 1 hour sampling and interpolate with the given method
+    ser_resampled = ser.resample("1h").interpolate(method)
+
+    # Check that none of the resampled values are NaN, except the first one
+    # which lies 1 minute before the first actual data point
+    assert np.isnan(ser_resampled.iloc[0])
+    assert not ser_resampled.iloc[1:].isna().any()
+
+    if method not in ["nearest", "zero"]:
+        # Check that the resampled values are close to the expected values
+        # except for methods with known inaccuracies
+        assert np.all(
+            np.isclose(ser_resampled.values[1:], np.arange(0.5, 4.5, 0.5), rtol=1.0e-1)
+        )
+
+
+@all_1d_no_arg_interpolation_methods
+def test_resample_interpolate_irregular_sampling(method):
+    # GH#21351
+    ser = Series(
+        np.linspace(0.0, 1.0, 5),
+        index=DatetimeIndex(
+            [
+                "2000-01-01 00:00:03",
+                "2000-01-01 00:00:22",
+                "2000-01-01 00:00:24",
+                "2000-01-01 00:00:31",
+                "2000-01-01 00:00:39",
+            ]
+        ),
+    )
+
+    # Resample to 5 second sampling and interpolate with the given method
+    ser_resampled = ser.resample("5s").interpolate(method)
+
+    # Check that none of the resampled values are NaN, except the first one
+    # which lies 3 seconds before the first actual data point
+    assert np.isnan(ser_resampled.iloc[0])
+    assert not ser_resampled.iloc[1:].isna().any()
+
+
 def test_raises_on_non_datetimelike_index():
     # this is a non datetimelike index
     xp = DataFrame()