Fixes grouped rank bug with nullable types.

doc/source/whatsnew/v2.1.0.rst

@@ -783,6 +783,7 @@ Plotting
 Groupby/resample/rolling
 ^^^^^^^^^^^^^^^^^^^^^^^^
 - Bug in :meth:`.DataFrameGroupBy.idxmin`, :meth:`.SeriesGroupBy.idxmin`, :meth:`.DataFrameGroupBy.idxmax`, :meth:`.SeriesGroupBy.idxmax` returns wrong dtype when used on an empty DataFrameGroupBy or SeriesGroupBy (:issue:`51423`)
+- Bug in :meth:`DataFrame.groupby` when using the rank function on nullable datatypes when passing ``na_option="bottom"`` or ``na_option="top"`` (:issue:`54206`)
 - Bug in :meth:`DataFrame.resample` and :meth:`Series.resample` in incorrectly allowing non-fixed ``freq`` when resampling on a :class:`TimedeltaIndex` (:issue:`51896`)
 - Bug in :meth:`DataFrame.resample` and :meth:`Series.resample` losing time zone when resampling empty data (:issue:`53664`)
 - Bug in :meth:`DataFrame.resample` and :meth:`Series.resample` where ``origin`` has no effect in resample when values are outside of axis  (:issue:`53662`)

pandas/core/arrays/masked.py

@@ -1492,6 +1492,9 @@ def _groupby_op(
         else:
             result_mask = np.zeros(ngroups, dtype=bool)
 
+        if op.how == "rank" and kwargs.get("na_option") in ["top", "bottom"]:
+            result_mask[:] = False
+
         res_values = op._cython_op_ndim_compat(
             self._data,
             min_count=min_count,

pandas/tests/groupby/test_rank.py

@@ -710,3 +710,14 @@ def test_rank_categorical():
 
     expected = df.astype(object).groupby("col1").rank()
     tm.assert_frame_equal(res, expected)
+
+
+@pytest.mark.parametrize("na_option", ["top", "bottom"])
+def test_groupby_op_with_nullables(na_option):
+    # GH 54206
+    df_ext = DataFrame({"x": [None]}, dtype="Float64")
+    result_ext = df_ext.groupby("x", dropna=False)["x"].rank(
+        method="min", na_option=na_option
+    )
+    expected_result_ext = Series([1.0], dtype="float64", name=result_ext.name)
+    tm.assert_series_equal(result_ext, expected_result_ext, check_dtype=False)