Backport PR pandas-dev#42801: DataFrame.drop silently does nothing if MultiIndex has duplicates

Author: phofl

doc/source/whatsnew/v1.3.2.rst

@@ -19,6 +19,7 @@ Fixed regressions
 - Regression in :meth:`DataFrame.from_records` with empty records (:issue:`42456`)
 - Fixed regression in :meth:`DataFrame.shift` where TypeError occurred when shifting DataFrame created by concatenation of slices and fills with values (:issue:`42719`)
 - Regression in :meth:`DataFrame.agg` when the ``func`` argument returned lists and ``axis=1`` (:issue:`42727`)
+- Regression in :meth:`DataFrame.drop` does nothing if :class:`MultiIndex` has duplicates and indexer is a tuple or list of tuples (:issue:`42771`)
 - Fixed regression where :meth:`pandas.read_csv` raised a ``ValueError`` when parameters ``names`` and ``prefix`` were both set to None (:issue:`42387`)
 - Fixed regression in comparisons between :class:`Timestamp` object and ``datetime64`` objects outside the implementation bounds for nanosecond ``datetime64`` (:issue:`42794`)
 -

pandas/core/generic.py

@@ -96,7 +96,10 @@
     ABCDataFrame,
     ABCSeries,
 )
-from pandas.core.dtypes.inference import is_hashable
+from pandas.core.dtypes.inference import (
+    is_hashable,
+    is_nested_list_like,
+)
 from pandas.core.dtypes.missing import (
     isna,
     notna,
@@ -4184,6 +4187,7 @@ def _drop_axis(
 
         # Case for non-unique axis
         else:
+            is_tuple_labels = is_nested_list_like(labels) or isinstance(labels, tuple)
             labels = ensure_object(com.index_labels_to_array(labels))
             if level is not None:
                 if not isinstance(axis, MultiIndex):
@@ -4193,9 +4197,14 @@ def _drop_axis(
                 # GH 18561 MultiIndex.drop should raise if label is absent
                 if errors == "raise" and indexer.all():
                     raise KeyError(f"{labels} not found in axis")
-            elif isinstance(axis, MultiIndex) and labels.dtype == "object":
+            elif (
+                isinstance(axis, MultiIndex)
+                and labels.dtype == "object"
+                and not is_tuple_labels
+            ):
                 # Set level to zero in case of MultiIndex and label is string,
                 #  because isin can't handle strings for MultiIndexes GH#36293
+                # In case of tuples we get dtype object but have to use isin GH#42771
                 indexer = ~axis.get_level_values(0).isin(labels)
             else:
                 indexer = ~axis.isin(labels)

pandas/tests/frame/methods/test_drop.py

@@ -456,6 +456,17 @@ def test_drop_with_non_unique_multiindex(self):
         expected = DataFrame([2], index=MultiIndex.from_arrays([["y"], ["j"]]))
         tm.assert_frame_equal(result, expected)
 
+    @pytest.mark.parametrize("indexer", [("a", "a"), [("a", "a")]])
+    def test_drop_tuple_with_non_unique_multiindex(self, indexer):
+        # GH#42771
+        idx = MultiIndex.from_product([["a", "b"], ["a", "a"]])
+        df = DataFrame({"x": range(len(idx))}, index=idx)
+        result = df.drop(index=[("a", "a")])
+        expected = DataFrame(
+            {"x": [2, 3]}, index=MultiIndex.from_tuples([("b", "a"), ("b", "a")])
+        )
+        tm.assert_frame_equal(result, expected)
+
     def test_drop_with_duplicate_columns(self):
         df = DataFrame(
             [[1, 5, 7.0], [1, 5, 7.0], [1, 5, 7.0]], columns=["bar", "a", "a"]