BUG/PERF: DataFrame.to_records inconsistent dtypes for MultiIndex (#47279)

DataFrame.to_records fix inconsistent dtypes with MultiIndex

Author: lukemanley

asv_bench/benchmarks/frame_methods.py

@@ -288,6 +288,26 @@ def time_values_mixed_wide(self):
         self.df_mixed_wide.values
 
 
+class ToRecords:
+    def setup(self):
+        N = 100_000
+        data = np.random.randn(N, 2)
+        mi = MultiIndex.from_arrays(
+            [
+                np.arange(N),
+                date_range("1970-01-01", periods=N, freq="ms"),
+            ]
+        )
+        self.df = DataFrame(data)
+        self.df_mi = DataFrame(data, index=mi)
+
+    def time_to_records(self):
+        self.df.to_records(index=True)
+
+    def time_to_records_multiindex(self):
+        self.df_mi.to_records(index=True)
+
+
 class Repr:
     def setup(self):
         nrows = 10000

doc/source/whatsnew/v1.5.0.rst

@@ -704,6 +704,7 @@ Performance improvements
 - Performance improvement in :meth:`.GroupBy.transform` for user-defined functions when only a single group exists (:issue:`44977`)
 - Performance improvement in :meth:`.GroupBy.apply` when grouping on a non-unique unsorted index (:issue:`46527`)
 - Performance improvement in :meth:`DataFrame.loc` and :meth:`Series.loc` for tuple-based indexing of a :class:`MultiIndex` (:issue:`45681`, :issue:`46040`, :issue:`46330`)
+- Performance improvement in :meth:`DataFrame.to_records` when the index is a :class:`MultiIndex` (:issue:`47263`)
 - Performance improvement in :attr:`MultiIndex.values` when the MultiIndex contains levels of type DatetimeIndex, TimedeltaIndex or ExtensionDtypes (:issue:`46288`)
 - Performance improvement in :func:`merge` when left and/or right are empty (:issue:`45838`)
 - Performance improvement in :meth:`DataFrame.join` when left and/or right are empty (:issue:`46015`)
@@ -765,6 +766,7 @@ Conversion
 - Bug in :func:`array` with ``FloatingDtype`` and values containing float-castable strings incorrectly raising (:issue:`45424`)
 - Bug when comparing string and datetime64ns objects causing ``OverflowError`` exception. (:issue:`45506`)
 - Bug in metaclass of generic abstract dtypes causing :meth:`DataFrame.apply` and :meth:`Series.apply` to raise for the built-in function ``type`` (:issue:`46684`)
+- Bug in :meth:`DataFrame.to_records` returning inconsistent numpy types if the index was a :class:`MultiIndex` (:issue:`47263`)
 - Bug in :meth:`DataFrame.to_dict` for ``orient="list"`` or ``orient="index"`` was not returning native types (:issue:`46751`)
 
 Strings

pandas/core/frame.py

@@ -2416,13 +2416,10 @@ def to_records(
                   dtype=[('I', 'S1'), ('A', '<i8'), ('B', '<f8')])
         """
         if index:
-            if isinstance(self.index, MultiIndex):
-                # array of tuples to numpy cols. copy copy copy
-                ix_vals = list(map(np.array, zip(*self.index._values)))
-            else:
-                # error: List item 0 has incompatible type "ArrayLike"; expected
-                # "ndarray"
-                ix_vals = [self.index.values]  # type: ignore[list-item]
+            ix_vals = [
+                np.asarray(self.index.get_level_values(i))
+                for i in range(self.index.nlevels)
+            ]
 
             arrays = ix_vals + [
                 np.asarray(self.iloc[:, i]) for i in range(len(self.columns))

pandas/tests/frame/methods/test_to_records.py

@@ -96,7 +96,7 @@ def test_to_records_index_name(self):
             + [np.asarray(df.iloc[:, i]) for i in range(3)],
             dtype={
                 "names": ["A", "level_1", "0", "1", "2"],
-                "formats": ["<U1", "<U1", "<f8", "<f8", "<f8"],
+                "formats": ["O", "O", "<f8", "<f8", "<f8"],
             },
         )
         tm.assert_numpy_array_equal(result, expected)
@@ -108,6 +108,33 @@ def test_to_records_with_unicode_index(self):
         expected = np.rec.array([("x", "y")], dtype=[("a", "O"), ("b", "O")])
         tm.assert_almost_equal(result, expected)
 
+    def test_to_records_index_dtype(self):
+        # GH 47263: consistent data types for Index and MultiIndex
+        df = DataFrame(
+            {
+                1: date_range("2022-01-01", periods=2),
+                2: date_range("2022-01-01", periods=2),
+                3: date_range("2022-01-01", periods=2),
+            }
+        )
+
+        expected = np.rec.array(
+            [
+                ("2022-01-01", "2022-01-01", "2022-01-01"),
+                ("2022-01-02", "2022-01-02", "2022-01-02"),
+            ],
+            dtype=[("1", "<M8[ns]"), ("2", "<M8[ns]"), ("3", "<M8[ns]")],
+        )
+
+        result = df.to_records(index=False)
+        tm.assert_almost_equal(result, expected)
+
+        result = df.set_index(1).to_records(index=True)
+        tm.assert_almost_equal(result, expected)
+
+        result = df.set_index([1, 2]).to_records(index=True)
+        tm.assert_almost_equal(result, expected)
+
     def test_to_records_with_unicode_column_names(self):
         # xref issue: https://github.com/numpy/numpy/issues/2407
         # Issue GH#11879. to_records used to raise an exception when used