BUG: MultiIndex.reindex with non-MultiIndex; Series constructor (#41707)

Author: jbrockmendel

doc/source/whatsnew/v1.3.0.rst

@@ -961,6 +961,7 @@ MultiIndex
 - Bug in :meth:`MultiIndex.equals` incorrectly returning ``True`` when :class:`MultiIndex` containing ``NaN`` even when they are differently ordered (:issue:`38439`)
 - Bug in :meth:`MultiIndex.intersection` always returning empty when intersecting with :class:`CategoricalIndex` (:issue:`38653`)
 - Bug in :meth:`MultiIndex.reindex` raising ``ValueError`` with empty MultiIndex and indexing only a specific level (:issue:`41170`)
+- Bug in :meth:`MultiIndex.reindex` raising ``TypeError`` when reindexing against a flat :class:`Index` (:issue:`41707`)
 
 I/O
 ^^^
@@ -1075,6 +1076,7 @@ Reshaping
 - Bug in :meth:`DataFrame.sort_values` not reshaping index correctly after sorting on columns, when ``ignore_index=True`` (:issue:`39464`)
 - Bug in :meth:`DataFrame.append` returning incorrect dtypes with combinations of ``ExtensionDtype`` dtypes (:issue:`39454`)
 - Bug in :meth:`DataFrame.append` returning incorrect dtypes with combinations of ``datetime64`` and ``timedelta64`` dtypes (:issue:`39574`)
+- Bug in :meth:`DataFrame.append` with a :class:`DataFrame` with a :class:`MultiIndex` and appending a :class:`Series` whose :class:`Index` is not a :class:`MultiIndex` (:issue:`41707`)
 - Bug in :meth:`DataFrame.pivot_table` returning a ``MultiIndex`` for a single value when operating on and empty ``DataFrame`` (:issue:`13483`)
 - Allow :class:`Index` to be passed to the :func:`numpy.all` function (:issue:`40180`)
 - Bug in :meth:`DataFrame.stack` not preserving ``CategoricalDtype`` in a ``MultiIndex`` (:issue:`36991`)
@@ -1129,6 +1131,7 @@ Other
 - Bug in :meth:`DataFrame.clip` not interpreting missing values as no threshold (:issue:`40420`)
 - Bug in :class:`Series` backed by :class:`DatetimeArray` or :class:`TimedeltaArray` sometimes failing to set the array's ``freq`` to ``None`` (:issue:`41425`)
 - Bug in creating a :class:`Series` from a ``range`` object that does not fit in the bounds of ``int64`` dtype (:issue:`30173`)
+- Bug in creating a :class:`Series` from a ``dict`` with all-tuple keys and an :class:`Index` that requires reindexing (:issue:`41707`)
 
 .. ---------------------------------------------------------------------------
 

pandas/core/frame.py

@@ -8921,10 +8921,7 @@ def append(
 
             index = Index([other.name], name=self.index.name)
             idx_diff = other.index.difference(self.columns)
-            try:
-                combined_columns = self.columns.append(idx_diff)
-            except TypeError:
-                combined_columns = self.columns.astype(object).append(idx_diff)
+            combined_columns = self.columns.append(idx_diff)
             other = (
                 other.reindex(combined_columns, copy=False)
                 .to_frame()

pandas/core/indexes/interval.py

@@ -774,9 +774,11 @@ def _get_indexer_pointwise(self, target: Index) -> tuple[np.ndarray, np.ndarray]
             except KeyError:
                 missing.append(i)
                 locs = np.array([-1])
-            except InvalidIndexError as err:
-                # i.e. non-scalar key
-                raise TypeError(key) from err
+            except InvalidIndexError:
+                # i.e. non-scalar key e.g. a tuple.
+                # see test_append_different_columns_types_raises
+                missing.append(i)
+                locs = np.array([-1])
 
             indexer.append(locs)
 

pandas/core/indexes/multi.py

@@ -2541,9 +2541,11 @@ def reindex(
             elif (indexer >= 0).all():
                 target = self.take(indexer)
             else:
-                # hopefully?
-                target = MultiIndex.from_tuples(target)
-
+                try:
+                    target = MultiIndex.from_tuples(target)
+                except TypeError:
+                    # not all tuples, see test_constructor_dict_multiindex_reindex_flat
+                    return target, indexer
         if (
             preserve_names
             and target.nlevels == self.nlevels

pandas/tests/indexes/multi/test_reindex.py

@@ -115,3 +115,14 @@ def test_reindex_empty_with_level(values):
     expected_indexer = np.array([], dtype=result_indexer.dtype)
     tm.assert_index_equal(result, expected)
     tm.assert_numpy_array_equal(result_indexer, expected_indexer)
+
+
+def test_reindex_not_all_tuples():
+    keys = [("i", "i"), ("i", "j"), ("j", "i"), "j"]
+    mi = MultiIndex.from_tuples(keys[:-1])
+    idx = Index(keys)
+    res, indexer = mi.reindex(idx)
+
+    tm.assert_index_equal(res, idx)
+    expected = np.array([0, 1, 2, -1], dtype=np.intp)
+    tm.assert_numpy_array_equal(indexer, expected)

pandas/tests/reshape/concat/test_append.py

@@ -184,18 +184,12 @@ def test_append_preserve_index_name(self):
                 dt.datetime(2013, 1, 3, 7, 12),
             ]
         ),
+        pd.MultiIndex.from_arrays(["A B C".split(), "D E F".split()]),
     ]
 
-    indexes_cannot_append_with_other = [
-        pd.MultiIndex.from_arrays(["A B C".split(), "D E F".split()])
-    ]
-
-    # error: Unsupported operand types for + ("List[Index]" and "List[MultiIndex]")
-    all_indexes = (
-        indexes_can_append + indexes_cannot_append_with_other  # type: ignore[operator]
+    @pytest.mark.parametrize(
+        "index", indexes_can_append, ids=lambda x: type(x).__name__
     )
-
-    @pytest.mark.parametrize("index", all_indexes, ids=lambda x: type(x).__name__)
     def test_append_same_columns_type(self, index):
         # GH18359
 
@@ -249,41 +243,6 @@ def test_append_different_columns_types(self, df_columns, series_index):
         )
         tm.assert_frame_equal(result, expected)
 
-    @pytest.mark.parametrize(
-        "index_can_append", indexes_can_append, ids=lambda x: type(x).__name__
-    )
-    @pytest.mark.parametrize(
-        "index_cannot_append_with_other",
-        indexes_cannot_append_with_other,
-        ids=lambda x: type(x).__name__,
-    )
-    def test_append_different_columns_types_raises(
-        self, index_can_append, index_cannot_append_with_other
-    ):
-        # GH18359
-        # Dataframe.append will raise if MultiIndex appends
-        # or is appended to a different index type
-        #
-        # See also test 'test_append_different_columns_types' above for
-        # appending without raising.
-
-        df = DataFrame([[1, 2, 3], [4, 5, 6]], columns=index_can_append)
-        ser = Series([7, 8, 9], index=index_cannot_append_with_other, name=2)
-        msg = (
-            r"Expected tuple, got (int|long|float|str|"
-            r"pandas._libs.interval.Interval)|"
-            r"object of type '(int|float|Timestamp|"
-            r"pandas._libs.interval.Interval)' has no len\(\)|"
-        )
-        with pytest.raises(TypeError, match=msg):
-            df.append(ser)
-
-        df = DataFrame([[1, 2, 3], [4, 5, 6]], columns=index_cannot_append_with_other)
-        ser = Series([7, 8, 9], index=index_can_append, name=2)
-
-        with pytest.raises(TypeError, match=msg):
-            df.append(ser)
-
     def test_append_dtype_coerce(self, sort):
 
         # GH 4993

pandas/tests/series/test_constructors.py

@@ -1717,6 +1717,14 @@ def test_constructor_dict_multiindex(self):
         result = result.reindex(index=expected.index)
         tm.assert_series_equal(result, expected)
 
+    def test_constructor_dict_multiindex_reindex_flat(self):
+        # construction involves reindexing with a MultiIndex corner case
+        data = {("i", "i"): 0, ("i", "j"): 1, ("j", "i"): 2, "j": np.nan}
+        expected = Series(data)
+
+        result = Series(expected[:-1].to_dict(), index=expected.index)
+        tm.assert_series_equal(result, expected)
+
     def test_constructor_dict_timedelta_index(self):
         # GH #12169 : Resample category data with timedelta index
         # construct Series from dict as data and TimedeltaIndex as index