diff --git a/doc/source/whatsnew/v2.0.0.rst b/doc/source/whatsnew/v2.0.0.rst index c2abb70988dad..f69c0a7e19f93 100644 --- a/doc/source/whatsnew/v2.0.0.rst +++ b/doc/source/whatsnew/v2.0.0.rst @@ -558,7 +558,7 @@ Missing MultiIndex ^^^^^^^^^^ -- Bug in :meth:`MultiIndex.get_indexer` not matching ``NaN`` values (:issue:`37222`) +- Bug in :meth:`MultiIndex.get_indexer` not matching ``NaN`` values (:issue:`29252`, :issue:`37222`, :issue:`38623`, :issue:`42883`, :issue:`43222`, :issue:`46173`, :issue:`48905`) - Bug in :meth:`MultiIndex.argsort` raising ``TypeError`` when index contains :attr:`NA` (:issue:`48495`) - Bug in :meth:`MultiIndex.difference` losing extension array dtype (:issue:`48606`) - Bug in :class:`MultiIndex.set_levels` raising ``IndexError`` when setting empty level (:issue:`48636`) diff --git a/pandas/tests/indexes/multi/test_indexing.py b/pandas/tests/indexes/multi/test_indexing.py index 337f91e0f89b4..324c347f2f088 100644 --- a/pandas/tests/indexes/multi/test_indexing.py +++ b/pandas/tests/indexes/multi/test_indexing.py @@ -933,3 +933,17 @@ def test_get_locs_reordering(keys, expected): result = idx.get_locs(keys) expected = np.array(expected, dtype=np.intp) tm.assert_numpy_array_equal(result, expected) + + +def test_get_indexer_for_multiindex_with_nans(nulls_fixture): + # GH37222 + idx1 = MultiIndex.from_product([["A"], [1.0, 2.0]], names=["id1", "id2"]) + idx2 = MultiIndex.from_product([["A"], [nulls_fixture, 2.0]], names=["id1", "id2"]) + + result = idx2.get_indexer(idx1) + expected = np.array([-1, 1], dtype=np.intp) + tm.assert_numpy_array_equal(result, expected) + + result = idx1.get_indexer(idx2) + expected = np.array([-1, 1], dtype=np.intp) + tm.assert_numpy_array_equal(result, expected) diff --git a/pandas/tests/indexes/multi/test_isin.py b/pandas/tests/indexes/multi/test_isin.py index 695458273d16e..bf003019d5387 100644 --- a/pandas/tests/indexes/multi/test_isin.py +++ b/pandas/tests/indexes/multi/test_isin.py @@ -13,6 +13,15 @@ def test_isin_nan(): ) +def test_isin_missing(nulls_fixture): + # GH48905 + mi1 = MultiIndex.from_tuples([(1, nulls_fixture)]) + mi2 = MultiIndex.from_tuples([(1, 1), (1, 2)]) + result = mi2.isin(mi1) + expected = np.array([False, False]) + tm.assert_numpy_array_equal(result, expected) + + def test_isin(): values = [("foo", 2), ("bar", 3), ("quux", 4)] diff --git a/pandas/tests/indexes/multi/test_join.py b/pandas/tests/indexes/multi/test_join.py index e6bec97aedb38..23d5325dde2bb 100644 --- a/pandas/tests/indexes/multi/test_join.py +++ b/pandas/tests/indexes/multi/test_join.py @@ -2,6 +2,7 @@ import pytest from pandas import ( + DataFrame, Index, Interval, MultiIndex, @@ -158,3 +159,21 @@ def test_join_overlapping_interval_level(): result = idx_1.join(idx_2, how="outer") tm.assert_index_equal(result, expected) + + +def test_join_multi_with_nan(): + # GH29252 + df1 = DataFrame( + data={"col1": [1.1, 1.2]}, + index=MultiIndex.from_product([["A"], [1.0, 2.0]], names=["id1", "id2"]), + ) + df2 = DataFrame( + data={"col2": [2.1, 2.2]}, + index=MultiIndex.from_product([["A"], [np.NaN, 2.0]], names=["id1", "id2"]), + ) + result = df1.join(df2) + expected = DataFrame( + data={"col1": [1.1, 1.2], "col2": [np.nan, 2.2]}, + index=MultiIndex.from_product([["A"], [1.0, 2.0]], names=["id1", "id2"]), + ) + tm.assert_frame_equal(result, expected) diff --git a/pandas/tests/indexes/multi/test_reindex.py b/pandas/tests/indexes/multi/test_reindex.py index 5d124c19cd865..7ef739056ada8 100644 --- a/pandas/tests/indexes/multi/test_reindex.py +++ b/pandas/tests/indexes/multi/test_reindex.py @@ -159,3 +159,15 @@ def test_reindex_limit_arg_with_multiindex(): match="limit argument only valid if doing pad, backfill or nearest reindexing", ): df.reindex(new_idx, fill_value=0, limit=1) + + +def test_reindex_with_none_in_nested_multiindex(): + # GH42883 + index = MultiIndex.from_tuples([(("a", None), 1), (("b", None), 2)]) + index2 = MultiIndex.from_tuples([(("b", None), 2), (("a", None), 1)]) + df1_dtype = pd.DataFrame([1, 2], index=index) + df2_dtype = pd.DataFrame([2, 1], index=index2) + + result = df1_dtype.reindex_like(df2_dtype) + expected = df2_dtype + tm.assert_frame_equal(result, expected) diff --git a/pandas/tests/indexes/multi/test_setops.py b/pandas/tests/indexes/multi/test_setops.py index e05fd68cab59a..d0345861d6778 100644 --- a/pandas/tests/indexes/multi/test_setops.py +++ b/pandas/tests/indexes/multi/test_setops.py @@ -4,6 +4,7 @@ import pandas as pd from pandas import ( CategoricalIndex, + DataFrame, Index, IntervalIndex, MultiIndex, @@ -548,6 +549,15 @@ def test_intersection_with_missing_values_on_both_sides(nulls_fixture): tm.assert_index_equal(result, expected) +def test_union_with_missing_values_on_both_sides(nulls_fixture): + # GH#38623 + mi1 = MultiIndex.from_arrays([[1, nulls_fixture]]) + mi2 = MultiIndex.from_arrays([[1, nulls_fixture, 3]]) + result = mi1.union(mi2) + expected = MultiIndex.from_arrays([[1, 3, nulls_fixture]]) + tm.assert_index_equal(result, expected) + + @pytest.mark.parametrize("dtype", ["float64", "Float64"]) @pytest.mark.parametrize("sort", [None, False]) def test_union_nan_got_duplicated(dtype, sort): @@ -651,7 +661,6 @@ def test_union_keep_dtype_precision(any_real_numeric_dtype): def test_union_keep_ea_dtype_with_na(any_numeric_ea_dtype): # GH#48498 - arr1 = Series([4, pd.NA], dtype=any_numeric_ea_dtype) arr2 = Series([1, pd.NA], dtype=any_numeric_ea_dtype) midx = MultiIndex.from_arrays([arr1, [2, 1]], names=["a", None]) @@ -695,3 +704,12 @@ def test_intersection_keep_ea_dtypes(val, any_numeric_ea_dtype): result = midx.intersection(midx2) expected = MultiIndex.from_arrays([Series([2], dtype=any_numeric_ea_dtype), [1]]) tm.assert_index_equal(result, expected) + + +def test_union_with_na_when_constructing_dataframe(): + # GH43222 + series1 = Series((1,), index=MultiIndex.from_tuples(((None, None),))) + series2 = Series((10, 20), index=MultiIndex.from_tuples(((None, None), ("a", "b")))) + result = DataFrame([series1, series2]) + expected = DataFrame({(np.nan, np.nan): [1.0, 10.0], ("a", "b"): [np.nan, 20.0]}) + tm.assert_frame_equal(result, expected) diff --git a/pandas/tests/indexing/multiindex/test_multiindex.py b/pandas/tests/indexing/multiindex/test_multiindex.py index 157f0de632e18..f5f58e7e818d9 100644 --- a/pandas/tests/indexing/multiindex/test_multiindex.py +++ b/pandas/tests/indexing/multiindex/test_multiindex.py @@ -216,3 +216,15 @@ def test_multiindex_repeated_keys(self): ], Series([1, 1, 2, 2], MultiIndex.from_arrays([["a", "a", "b", "b"]])), ) + + def test_multiindex_with_na_missing_key(self): + # GH46173 + df = DataFrame.from_dict( + { + ("foo",): [1, 2, 3], + ("bar",): [5, 6, 7], + (None,): [8, 9, 0], + } + ) + with pytest.raises(KeyError, match="missing_key"): + df[[("missing_key",)]]