BUG: Slice Arrow buffer before passing it to numpy (pandas-dev#40896)

Fix mask buffer in test.

Author: ThomasBlauthQC

pandas/core/arrays/_arrow_utils.py

@@ -33,8 +33,6 @@ def pyarrow_array_to_numpy_and_mask(arr, dtype):
     # Since Arrow buffers might contain padding and the data might be offset,
     # the buffer gets sliced here before handing it to numpy.
     # See also https://github.com/pandas-dev/pandas/issues/40896
-    # offset = arr.offset * arr.type.bit_width // 8
-    # length = len(arr) * arr.type.bit_width // 8
     offset = arr.offset * dtype.itemsize
     length = len(arr) * dtype.itemsize
     data_buf = buflist[1][offset : offset + length]

pandas/tests/arrays/test_arrow_utils.py

@@ -25,7 +25,7 @@ def test_pyarrow_array_to_numpy_and_mask():
 
     data, mask = pyarrow_array_to_numpy_and_mask(pa_array, dtype)
     tm.assert_numpy_array_equal(data, np_expected)
-    assert (mask == mask_expected).all()
+    tm.assert_numpy_array_equal(mask, mask_expected)
 
     mask_buffer = pa_array.buffers()[0]
     data_buffer = pa_array.buffers()[1].to_pybytes()
@@ -41,18 +41,19 @@ def test_pyarrow_array_to_numpy_and_mask():
     pa_array_trail.validate()
     data, mask = pyarrow_array_to_numpy_and_mask(pa_array_trail, dtype)
     tm.assert_numpy_array_equal(data, np_expected)
-    assert (mask == mask_expected).all()
+    tm.assert_numpy_array_equal(mask, mask_expected)
 
     # Add offset to the buffer.
     offset = b"\x00" * (pa_array.type.bit_width // 8)
     data_buffer_offset = pa.py_buffer(offset + data_buffer)
+    mask_buffer_offset = pa.py_buffer(b"\x0F")
     pa_array_offset = pa.Array.from_buffers(
         type=pa_array.type,
         length=len(pa_array),
-        buffers=[mask_buffer, data_buffer_offset],
+        buffers=[mask_buffer_offset, data_buffer_offset],
         offset=pa_array.offset + 1,
     )
     pa_array_offset.validate()
     data, mask = pyarrow_array_to_numpy_and_mask(pa_array_offset, dtype)
     tm.assert_numpy_array_equal(data, np_expected)
-    assert (mask == mask_expected).all()
+    tm.assert_numpy_array_equal(mask, mask_expected)