BUG: Raise ValueError with nan in timeaware windows

doc/source/whatsnew/v1.2.0.rst

@@ -408,6 +408,7 @@ Groupby/resample/rolling
 - Bug in :meth:`DataFrameGroupBy.apply` raising error with ``np.nan`` group(s) when ``dropna=False`` (:issue:`35889`)
 - Bug in :meth:`Rolling.sum()` returned wrong values when dtypes where mixed between float and integer and axis was equal to one (:issue:`20649`, :issue:`35596`)
 - Bug in :meth:`Rolling.count` returned ``np.nan`` with :class:`pandas.api.indexers.FixedForwardWindowIndexer` as window, ``min_periods=0`` and only missing values in window (:issue:`35579`)
+- Bug in :meth:`DataFrameGroupBy.rolling` returned wrong values with timeaware window containing ``NaN``. Raises ``ValueError`` because windows are not monotonic now (:issue:`34617`)
 
 Reshaping
 ^^^^^^^^^

pandas/core/window/rolling.py

@@ -1968,10 +1968,13 @@ def _validate_monotonic(self):
         Validate monotonic (increasing or decreasing).
         """
         if not (self._on.is_monotonic_increasing or self._on.is_monotonic_decreasing):
-            formatted = self.on
-            if self.on is None:
-                formatted = "index"
-            raise ValueError(f"{formatted} must be monotonic")
+            self._raise_monotonic_error()
+
+    def _raise_monotonic_error(self):
+        formatted = self.on
+        if self.on is None:
+            formatted = "index"
+        raise ValueError(f"{formatted} must be monotonic")
 
     def _validate_freq(self):
         """
@@ -2287,8 +2290,8 @@ def _gotitem(self, key, ndim, subset=None):
     def _validate_monotonic(self):
         """
         Validate that on is monotonic;
-        we don't care for groupby.rolling
-        because we have already validated at a higher
-        level.
+        in this case we have to check only for nans, because
+        monotonicy was already validated at a higher level.
         """
-        pass
+        if self._on.hasnans:
+            self._raise_monotonic_error()

pandas/tests/window/test_grouper.py

@@ -428,3 +428,20 @@ def test_groupby_rolling_empty_frame(self):
         result = expected.groupby(["s1", "s2"]).rolling(window=1).sum()
         expected.index = pd.MultiIndex.from_tuples([], names=["s1", "s2", None])
         tm.assert_frame_equal(result, expected)
+
+    @pytest.mark.parametrize(
+        ("rollings", "key"), [({"on": "a"}, "a"), ({"on": None}, "index")]
+    )
+    def test_groupby_rolling_nans_in_index(self, rollings, key):
+        # GH: 34617
+        df = pd.DataFrame(
+            {
+                "a": pd.to_datetime(["2020-06-01 12:00", "2020-06-01 14:00", np.nan]),
+                "b": [1, 2, 3],
+                "c": [1, 1, 1],
+            }
+        )
+        if key == "index":
+            df.set_index("a", inplace=True)
+        with pytest.raises(ValueError, match=f"{key} must be monotonic"):
+            df.groupby("c").rolling("60min", **rollings)