fix tests which trigger NullKeyWarning

this will help with PDEP-11 (pandas-dev#53094) as an intermediate step to identify tests that will fail under the default value

Author: tehunter

pandas/tests/groupby/test_categorical.py

@@ -192,7 +192,7 @@ def test_basic_cut_grouping():
     # GH 9603
     df = DataFrame({"a": [1, 0, 0, 0]})
     c = pd.cut(df.a, [0, 1, 2, 3, 4], labels=Categorical(list("abcd")))
-    result = df.groupby(c, observed=False).apply(len)
+    result = df.groupby(c, observed=False, dropna=True).apply(len)
 
     exp_index = CategoricalIndex(c.values.categories, ordered=c.values.ordered)
     expected = Series([1, 0, 0, 0], index=exp_index)
@@ -568,7 +568,7 @@ def test_observed_groups_with_nan(observed):
             "vals": [1, 2, 3],
         }
     )
-    g = df.groupby("cat", observed=observed)
+    g = df.groupby("cat", observed=observed, dropna=True)
     result = g.groups
     if observed:
         expected = {"a": Index([0, 2], dtype="int64")}
@@ -587,7 +587,7 @@ def test_observed_nth():
     ser = Series([1, 2, 3])
     df = DataFrame({"cat": cat, "ser": ser})
 
-    result = df.groupby("cat", observed=False)["ser"].nth(0)
+    result = df.groupby("cat", observed=False, dropna=True)["ser"].nth(0)
     expected = df["ser"].iloc[[0]]
     tm.assert_series_equal(result, expected)
 
@@ -597,7 +597,7 @@ def test_dataframe_categorical_with_nan(observed):
     s1 = Categorical([np.nan, "a", np.nan, "a"], categories=["a", "b", "c"])
     s2 = Series([1, 2, 3, 4])
     df = DataFrame({"s1": s1, "s2": s2})
-    result = df.groupby("s1", observed=observed).first().reset_index()
+    result = df.groupby("s1", observed=observed, dropna=True).first().reset_index()
     if observed:
         expected = DataFrame(
             {"s1": Categorical(["a"], categories=["a", "b", "c"]), "s2": [2]}
@@ -768,7 +768,9 @@ def test_categorical_series(series, data):
     # Group the given series by a series with categorical data type such that group A
     # takes indices 0 and 3 and group B indices 1 and 2, obtaining the values mapped in
     # the given data.
-    groupby = series.groupby(Series(list("ABBA"), dtype="category"), observed=False)
+    groupby = series.groupby(
+        Series(list("ABBA"), dtype="category"), observed=False, dropna=True
+    )
     result = groupby.aggregate(list)
     expected = Series(data, index=CategoricalIndex(data.keys()))
     tm.assert_series_equal(result, expected)
@@ -973,7 +975,7 @@ def test_groupby_empty_with_category():
     # test fix for when group by on None resulted in
     # coercion of dtype categorical -> float
     df = DataFrame({"A": [None] * 3, "B": Categorical(["train", "train", "test"])})
-    result = df.groupby("A").first()["B"]
+    result = df.groupby("A", dropna=True).first()["B"]
     expected = Series(
         Categorical([], categories=["test", "train"]),
         index=Series([], dtype="object", name="A"),

pandas/tests/groupby/test_groupby.py

@@ -140,9 +140,9 @@ def test_len():
 def test_len_nan_group():
     # issue 11016
     df = DataFrame({"a": [np.nan] * 3, "b": [1, 2, 3]})
-    assert len(df.groupby("a")) == 0
+    assert len(df.groupby("a", dropna=True)) == 0
     assert len(df.groupby("b")) == 3
-    assert len(df.groupby(["a", "b"])) == 0
+    assert len(df.groupby(["a", "b"], dropna=True)) == 0
 
 
 def test_groupby_timedelta_median():
@@ -922,6 +922,7 @@ def test_groupby_complex_numbers():
     tm.assert_frame_equal(result, expected)
 
 
+@pytest.mark.filterwarnings("ignore::pandas.errors.NullKeyWarning")
 def test_groupby_series_indexed_differently():
     s1 = Series(
         [5.0, -9.0, 4.0, 100.0, -5.0, 55.0, 6.7],
@@ -1215,7 +1216,7 @@ def test_groupby_nat_exclude():
             "str": [np.nan, "a", np.nan, "a", np.nan, "a", np.nan, "b"],
         }
     )
-    grouped = df.groupby("dt")
+    grouped = df.groupby("dt", dropna=True)
 
     expected = [
         RangeIndex(start=1, stop=13, step=6),
@@ -1253,7 +1254,7 @@ def test_groupby_nat_exclude():
     assert nan_df["nat"].dtype == "datetime64[s]"
 
     for key in ["nan", "nat"]:
-        grouped = nan_df.groupby(key)
+        grouped = nan_df.groupby(key, dropna=True)
         assert grouped.groups == {}
         assert grouped.ngroups == 0
         assert grouped.indices == {}
@@ -1266,7 +1267,7 @@ def test_groupby_nat_exclude():
 def test_groupby_two_group_keys_all_nan():
     # GH #36842: Grouping over two group keys shouldn't raise an error
     df = DataFrame({"a": [np.nan, np.nan], "b": [np.nan, np.nan], "c": [1, 2]})
-    result = df.groupby(["a", "b"]).indices
+    result = df.groupby(["a", "b"], dropna=True).indices
     assert result == {}
 
 
@@ -2050,7 +2051,7 @@ def test_groupby_only_none_group():
     # see GH21624
     # this was crashing with "ValueError: Length of passed values is 1, index implies 0"
     df = DataFrame({"g": [None], "x": 1})
-    actual = df.groupby("g")["x"].transform("sum")
+    actual = df.groupby("g", dropna=True)["x"].transform("sum")
     expected = Series([np.nan], name="x")
 
     tm.assert_series_equal(actual, expected)
@@ -2295,7 +2296,7 @@ def test_groupby_mean_duplicate_index(rand_series_with_duplicate_datetimeindex):
 def test_groupby_all_nan_groups_drop():
     # GH 15036
     s = Series([1, 2, 3], [np.nan, np.nan, np.nan])
-    result = s.groupby(s.index).sum()
+    result = s.groupby(s.index, dropna=True).sum()
     expected = Series([], index=Index([], dtype=np.float64), dtype=np.int64)
     tm.assert_series_equal(result, expected)
 
@@ -2459,7 +2460,7 @@ def test_groupby_none_in_first_mi_level():
     # GH#47348
     arr = [[None, 1, 0, 1], [2, 3, 2, 3]]
     ser = Series(1, index=MultiIndex.from_arrays(arr, names=["a", "b"]))
-    result = ser.groupby(level=[0, 1]).sum()
+    result = ser.groupby(level=[0, 1], dropna=True).sum()
     expected = Series(
         [1, 2], MultiIndex.from_tuples([(0.0, 2), (1.0, 3)], names=["a", "b"])
     )
@@ -2632,9 +2633,9 @@ def test_groupby_method_drop_na(method):
     df = DataFrame({"A": ["a", np.nan, "b", np.nan, "c"], "B": range(5)})
 
     if method == "nth":
-        result = getattr(df.groupby("A"), method)(n=0)
+        result = getattr(df.groupby("A", dropna=True), method)(n=0)
     else:
-        result = getattr(df.groupby("A"), method)()
+        result = getattr(df.groupby("A", dropna=True), method)()
 
     if method in ["first", "last"]:
         expected = DataFrame({"B": [0, 2, 4]}).set_index(

pandas/tests/groupby/test_grouping.py

@@ -691,7 +691,7 @@ def test_groupby_level_with_nas(self, sort):
 
         # factorizing doesn't confuse things
         s = Series(np.arange(8.0), index=index)
-        result = s.groupby(level=0, sort=sort).sum()
+        result = s.groupby(level=0, sort=sort, dropna=True).sum()
         expected = Series([6.0, 18.0], index=[0.0, 1.0])
         tm.assert_series_equal(result, expected)
 
@@ -817,7 +817,7 @@ def test_groupby_level_index_value_all_na(self):
         df = DataFrame(
             [["x", np.nan, 10], [None, np.nan, 20]], columns=["A", "B", "C"]
         ).set_index(["A", "B"])
-        result = df.groupby(level=["A", "B"]).sum()
+        result = df.groupby(level=["A", "B"], dropna=True).sum()
         expected = DataFrame(
             data=[],
             index=MultiIndex(

pandas/tests/groupby/test_indexing.py

@@ -294,6 +294,7 @@ def test_groupby_duplicated_columns(func):
     tm.assert_frame_equal(result, expected)
 
 
+@pytest.mark.filterwarnings("ignore::pandas.errors.NullKeyWarning")
 def test_groupby_get_nonexisting_groups():
     # GH#32492
     df = pd.DataFrame(

pandas/tests/groupby/test_reductions.py

@@ -656,32 +656,34 @@ def test_multifunc_skipna(func, values, dtype, result_dtype, skipna):
     tm.assert_series_equal(result, expected)
 
 
-def test_cython_median():
+def test_cython_median(dropna):
     arr = np.random.default_rng(2).standard_normal(1000)
     arr[::2] = np.nan
     df = DataFrame(arr)
 
     labels = np.random.default_rng(2).integers(0, 50, size=1000).astype(float)
     labels[::17] = np.nan
 
-    result = df.groupby(labels).median()
-    exp = df.groupby(labels).agg(np.nanmedian)
+    result = df.groupby(labels, dropna=dropna).median()
+    exp = df.groupby(labels, dropna=dropna).agg(np.nanmedian)
     tm.assert_frame_equal(result, exp)
 
     df = DataFrame(np.random.default_rng(2).standard_normal((1000, 5)))
-    rs = df.groupby(labels).agg(np.median)
-    xp = df.groupby(labels).median()
+    rs = df.groupby(labels, dropna=dropna).agg(np.median)
+    xp = df.groupby(labels, dropna=dropna).median()
     tm.assert_frame_equal(rs, xp)
 
 
-def test_median_empty_bins(observed):
+def test_median_empty_bins(observed, dropna):
     df = DataFrame(np.random.default_rng(2).integers(0, 44, 500))
 
     grps = range(0, 55, 5)
     bins = pd.cut(df[0], grps)
 
-    result = df.groupby(bins, observed=observed).median()
-    expected = df.groupby(bins, observed=observed).agg(lambda x: x.median())
+    result = df.groupby(bins, observed=observed, dropna=dropna).median()
+    expected = df.groupby(bins, observed=observed, dropna=dropna).agg(
+        lambda x: x.median()
+    )
     tm.assert_frame_equal(result, expected)
 
 
@@ -1069,6 +1071,7 @@ def test_max_nan_bug():
 
 
 @pytest.mark.slow
+@pytest.mark.filterwarnings("ignore::pandas.errors.NullKeyWarning")
 @pytest.mark.parametrize("with_nan", [True, False])
 @pytest.mark.parametrize("keys", [["joe"], ["joe", "jim"]])
 def test_series_groupby_nunique(sort, dropna, as_index, with_nan, keys):