BUG: Raise ValueError with nan in timeaware windows

doc/source/whatsnew/v1.2.0.rst

@@ -460,6 +460,7 @@ Groupby/resample/rolling
 - Bug in :meth:`DataFrameGroupBy.ffill` and :meth:`DataFrameGroupBy.bfill` where a ``NaN`` group would return filled values instead of ``NaN`` when ``dropna=True`` (:issue:`34725`)
 - Bug in :meth:`RollingGroupby.count` where a ``ValueError`` was raised when specifying the ``closed`` parameter (:issue:`35869`)
 - Bug in :meth:`DataFrame.groupby.rolling` returning wrong values with partial centered window (:issue:`36040`).
+- Bug in :meth:`DataFrameGroupBy.rolling` returned wrong values with timeaware window containing ``NaN``. Raises ``ValueError`` because windows are not monotonic now (:issue:`34617`)
 
 Reshaping
 ^^^^^^^^^

pandas/core/window/rolling.py

@@ -2085,10 +2085,13 @@ def _validate_monotonic(self):
         Validate monotonic (increasing or decreasing).
         """
         if not (self._on.is_monotonic_increasing or self._on.is_monotonic_decreasing):
-            formatted = self.on
-            if self.on is None:
-                formatted = "index"
-            raise ValueError(f"{formatted} must be monotonic")
+            self._raise_monotonic_error()
+
+    def _raise_monotonic_error(self):
+        formatted = self.on
+        if self.on is None:
+            formatted = "index"
+        raise ValueError(f"{formatted} must be monotonic")
 
     def _validate_freq(self):
         """
@@ -2323,3 +2326,12 @@ def _get_window_indexer(self, window: int) -> GroupbyIndexer:
             indexer_kwargs=indexer_kwargs,
         )
         return window_indexer
+
+    def _validate_monotonic(self):
+        """
+        Validate that on is monotonic;
+        in this case we have to check only for nans, because
+        monotonicy was already validated at a higher level.
+        """
+        if self._on.hasnans:
+            self._raise_monotonic_error()

pandas/tests/window/test_grouper.py

@@ -549,3 +549,20 @@ def test_groupby_rolling_sem(self, func, kwargs):
             ),
         )
         tm.assert_frame_equal(result, expected)
+
+    @pytest.mark.parametrize(
+        ("rollings", "key"), [({"on": "a"}, "a"), ({"on": None}, "index")]
+    )
+    def test_groupby_rolling_nans_in_index(self, rollings, key):
+        # GH: 34617
+        df = pd.DataFrame(
+            {
+                "a": pd.to_datetime(["2020-06-01 12:00", "2020-06-01 14:00", np.nan]),
+                "b": [1, 2, 3],
+                "c": [1, 1, 1],
+            }
+        )
+        if key == "index":
+            df = df.set_index("a")
+        with pytest.raises(ValueError, match=f"{key} must be monotonic"):
+            df.groupby("c").rolling("60min", **rollings)