PERF: DataFrame constructor (#43142)

Author: jbrockmendel

pandas/core/internals/construction.py

@@ -121,11 +121,30 @@ def arrays_to_mgr(
 
         # don't force copy because getting jammed in an ndarray anyway
         arrays = _homogenize(arrays, index, dtype)
+        # _homogenize ensures
+        #  - all(len(x) == len(index) for x in arrays)
+        #  - all(x.ndim == 1 for x in arrays)
+        #  - all(isinstance(x, (np.ndarray, ExtensionArray)) for x in arrays)
+        #  - all(type(x) is not PandasArray for x in arrays)
 
     else:
         index = ensure_index(index)
 
+        # Reached via DataFrame._from_arrays; we do validation here
+        for arr in arrays:
+            if (
+                not isinstance(arr, (np.ndarray, ExtensionArray))
+                or arr.ndim != 1
+                or len(arr) != len(index)
+            ):
+                raise ValueError(
+                    "Arrays must be 1-dimensional np.ndarray or ExtensionArray "
+                    "with length matching len(index)"
+                )
+
     columns = ensure_index(columns)
+    if len(columns) != len(arrays):
+        raise ValueError("len(arrays) must match len(columns)")
 
     # from BlockManager perspective
     axes = [columns, index]
@@ -581,6 +600,7 @@ def _homogenize(data, index: Index, dtype: DtypeObj | None) -> list[ArrayLike]:
             val = sanitize_array(
                 val, index, dtype=dtype, copy=False, raise_cast_failure=False
             )
+            com.require_length_match(val, index)
 
         homogenized.append(val)
 

pandas/core/internals/managers.py

@@ -1813,15 +1813,20 @@ def create_block_manager_from_column_arrays(
     axes: list[Index],
     consolidate: bool = True,
 ) -> BlockManager:
-    # Assertions disabled for performance
+    # Assertions disabled for performance (caller is responsible for verifying)
     # assert isinstance(axes, list)
     # assert all(isinstance(x, Index) for x in axes)
+    # assert all(x.ndim == 1 for x in arrays)
+    # assert all(len(x) == len(axes[1]) for x in arrays)
+    # assert len(arrays) == len(axes[0])
+    # These last three are sufficient to allow us to safely pass
+    #  verify_integrity=False below.
 
     arrays = [extract_array(x, extract_numpy=True) for x in arrays]
 
     try:
         blocks = _form_blocks(arrays, consolidate)
-        mgr = BlockManager(blocks, axes)
+        mgr = BlockManager(blocks, axes, verify_integrity=False)
     except ValueError as e:
         raise construction_error(len(arrays), arrays[0].shape, axes, e)
     if consolidate:

pandas/tests/frame/constructors/test_from_records.py

@@ -188,8 +188,7 @@ def test_from_records_bad_index_column(self):
         # should fail
         msg = "|".join(
             [
-                r"Shape of passed values is \(10, 3\), indices imply \(1, 3\)",
-                "Passed arrays should have the same length as the rows Index: 10 vs 1",
+                r"Length of values \(10\) does not match length of index \(1\)",
             ]
         )
         with pytest.raises(ValueError, match=msg):
@@ -268,8 +267,7 @@ def test_from_records_to_records(self):
         # wrong length
         msg = "|".join(
             [
-                r"Shape of passed values is \(2, 3\), indices imply \(1, 3\)",
-                "Passed arrays should have the same length as the rows Index: 2 vs 1",
+                r"Length of values \(2\) does not match length of index \(1\)",
             ]
         )
         with pytest.raises(ValueError, match=msg):

pandas/tests/frame/test_constructors.py

@@ -2222,8 +2222,7 @@ def test_construct_from_listlikes_mismatched_lengths(self):
         # invalid (shape)
         msg = "|".join(
             [
-                r"Shape of passed values is \(6, 2\), indices imply \(3, 2\)",
-                "Passed arrays should have the same length as the rows Index",
+                r"Length of values \(6\) does not match length of index \(3\)",
             ]
         )
         msg2 = "will be changed to match the behavior"
@@ -2791,6 +2790,17 @@ def test_construction_from_ndarray_datetimelike(self):
         df = DataFrame(arr)
         assert all(isinstance(arr, DatetimeArray) for arr in df._mgr.arrays)
 
+    def test_construction_from_ndarray_with_eadtype_mismatched_columns(self):
+        arr = np.random.randn(10, 2)
+        dtype = pd.array([2.0]).dtype
+        msg = r"len\(arrays\) must match len\(columns\)"
+        with pytest.raises(ValueError, match=msg):
+            DataFrame(arr, columns=["foo"], dtype=dtype)
+
+        arr2 = pd.array([2.0, 3.0, 4.0])
+        with pytest.raises(ValueError, match=msg):
+            DataFrame(arr2, columns=["foo", "bar"])
+
 
 def get1(obj):
     if isinstance(obj, Series):

pandas/tests/io/json/test_pandas.py

@@ -317,9 +317,7 @@ def test_roundtrip_mixed(self, request, orient, convert_axes, numpy):
                 '"data":[[1.0,"1"],[2.0,"2"],[null,"3"]]}',
                 "|".join(
                     [
-                        r"Shape of passed values is \(3, 2\), indices imply \(2, 2\)",
-                        "Passed arrays should have the same length as the rows Index: "
-                        "3 vs 2 rows",
+                        r"Length of values \(3\) does not match length of index \(2\)",
                     ]
                 ),
                 "split",