Bitmask Backed MaskedArray

.pre-commit-config.yaml

@@ -70,19 +70,6 @@ repos:
     -   id: fix-encoding-pragma
         args: [--remove]
     -   id: trailing-whitespace
--   repo: https://github.com/cpplint/cpplint
-    rev: 1.6.1
-    hooks:
-    -   id: cpplint
-        exclude: ^pandas/_libs/include/pandas/vendored/klib
-        args: [
-            --quiet,
-            '--extensions=c,h',
-            '--headers=h',
-            --recursive,
-            --linelength=88,
-            '--filter=-readability/casting,-runtime/int,-build/include_subdir,-readability/fn_size'
-        ]
 -   repo: https://github.com/pylint-dev/pylint
     rev: v3.0.0a6
     hooks:

pandas/_libs/arrays.pyx

@@ -7,10 +7,28 @@ import numpy as np
 
 cimport numpy as cnp
 from cpython cimport PyErr_Clear
-from numpy cimport ndarray
+from numpy cimport (
+    int8_t,
+    int64_t,
+    ndarray,
+    uint8_t,
+)
 
 cnp.import_array()
 
+from libc.stdlib cimport (
+    free,
+    malloc,
+)
+
+from pandas._libs.lib import is_list_like
+
+
+cdef extern from "pandas/vendored/nanoarrow.h":
+    int8_t ArrowBitGet(const uint8_t*, int64_t)
+    void ArrowBitSet(uint8_t*, int64_t)
+    void ArrowBitClear(uint8_t*, int64_t)
+
 
 @cython.freelist(16)
 cdef class NDArrayBacked:
@@ -189,3 +207,45 @@ cdef class NDArrayBacked:
         new_values = [obj._ndarray for obj in to_concat]
         new_arr = cnp.PyArray_Concatenate(new_values, axis)
         return to_concat[0]._from_backing_data(new_arr)
+
+
+cdef class BitMaskArray:
+    cdef array_len
+    cdef uint8_t* validity_buffer
+
+    def __cinit__(self, np_array):
+        self.array_len = len(np_array)
+        nbytes = len(np_array) // 8 + 1
+        self.validity_buffer = <uint8_t *>malloc(nbytes)
+        for index, value in enumerate(np_array):
+            self[index] = value
+
+    def __dealloc__(self):
+        free(self.validity_buffer)
+
+    def __setitem__(self, key, value):
+        if is_list_like(key):
+            for k in key:
+                if value:
+                    ArrowBitSet(self.validity_buffer, k)
+                else:
+                    ArrowBitClear(self.validity_buffer, k)
+        else:
+            if value:
+                ArrowBitSet(self.validity_buffer, key)
+            else:
+                ArrowBitClear(self.validity_buffer, key)
+
+    def __getitem__(self, key):
+        return bool(ArrowBitGet(self.validity_buffer, key))
+
+    def __invert__(self):
+        return ~self.to_numpy()
+
+    def to_numpy(self) -> ndarray:
+        cdef ndarray[uint8_t] result
+        result = np.empty(self.array_len, dtype=bool)
+        for i in range(self.array_len):
+            result[i] = self[i]
+
+        return result