Assorted cleanups

Author: WillAyd

pandas/core/arrays/list_.py

@@ -74,7 +74,7 @@ class ListDtype(ArrowDtype):
     An ExtensionDtype suitable for storing homogeneous lists of data.
     """
 
-    _is_immutable = True  # TODO(wayd): should we allow mutability?
+    _is_immutable = True
 
     def __init__(self, value_dtype: pa.DataType) -> None:
         super().__init__(pa.large_list(value_dtype))
@@ -100,10 +100,7 @@ def name(self) -> str:  # type: ignore[override]
         """
         A string identifying the data type.
         """
-        # TODO: reshaping tests require the name list to match the large_list
-        # implementation; assumedly there are some astype(str(dtype)) casts
-        # going on. Should fix so this can just be "list[...]" for end user
-        return f"large_list[{self.pyarrow_dtype.value_type!s}]"
+        return f"list[{self.pyarrow_dtype.value_type!s}]"
 
     @property
     def kind(self) -> str:
@@ -124,7 +121,6 @@ def construct_array_type(cls) -> type_t[ListArray]:
         return ListArray
 
     def _get_common_dtype(self, dtypes: list[DtypeObj]) -> DtypeObj | None:
-        # TODO(wayd): should we implemented value type support?
         for dtype in dtypes:
             if (
                 isinstance(dtype, ListDtype)
@@ -153,8 +149,7 @@ def __init__(
                 if isinstance(values, (pa.Array, pa.ChunkedArray)):
                     parent_type = values.type
                     if not isinstance(parent_type, (pa.ListType, pa.LargeListType)):
-                        # Ideally could cast here, but I don't think pyarrow implements
-                        # many list casts
+                        # TODO: maybe implement native casts in pyarrow
                         new_values = [
                             [x.as_py()] if x.is_valid else None for x in values
                         ]
@@ -164,12 +159,10 @@ def __init__(
                 else:
                     value_type = pa.array(values).type.value_type
 
-                # Internally always use large_string instead of string
                 if value_type == pa.string():
                     value_type = pa.large_string()
 
             if not isinstance(values, pa.ChunkedArray):
-                # To support NA, we need to create an Array first :-(
                 arr = pa.array(values, type=pa.large_list(value_type), from_pandas=True)
                 self._pa_array = pa.chunked_array(arr, type=pa.large_list(value_type))
             else:
@@ -200,8 +193,6 @@ def _from_sequence(cls, scalars, *, dtype=None, copy: bool = False):
             values = pa.array(scalars, from_pandas=True)
 
         if values.type == "null" and dtype is not None:
-            # TODO: the sequencing here seems wrong; just making the tests pass for now
-            # but this needs a comprehensive review
             pa_type = string_to_pyarrow_type(str(dtype))
             values = pa.array(values, type=pa_type)
 
@@ -232,8 +223,6 @@ def _box_pa(
         return cls._box_pa_array(value, pa_type)
 
     def __getitem__(self, item):
-        # PyArrow does not support NumPy's selection with an equal length
-        # mask, so let's convert those to integral positions if needed
         if isinstance(item, (np.ndarray, ExtensionArray)):
             if is_bool_dtype(item.dtype):
                 mask_len = len(item)
@@ -305,9 +294,6 @@ def _empty(cls, shape: Shape, dtype: ExtensionDtype):
         ExtensionDtype.empty
             ExtensionDtype.empty is the 'official' public version of this API.
         """
-        # Implementer note: while ExtensionDtype.empty is the public way to
-        # call this method, it is still required to implement this `_empty`
-        # method as well (it is called internally in pandas)
         if isinstance(shape, tuple):
             if len(shape) > 1:
                 raise ValueError("ListArray may only be 1-D")
@@ -334,9 +320,9 @@ def __eq__(self, other):
         elif isinstance(other, (pa.ListScalar, pa.LargeListScalar)):
             from pandas.arrays import BooleanArray
 
-            # TODO: pyarrow.compute does not implement broadcasting equality
-            # for an array of lists to a listscalar
-            # TODO: pyarrow doesn't compare missing values as missing???
+            # TODO: pyarrow.compute does not implement equal for lists
+            # https://github.com/apache/arrow/issues/45167
+            # TODO: pyarrow doesn't compare missing values in Python as missing???
             # arr = pa.array([1, 2, None])
             # pc.equal(arr, arr[2]) returns all nulls but
             # arr[2] == arr[2] returns True

pandas/core/internals/construction.py

@@ -13,7 +13,6 @@
 
 import numpy as np
 from numpy import ma
-import pyarrow as pa
 
 from pandas._config import using_string_dtype
 
@@ -462,8 +461,6 @@ def treat_as_nested(data, dtype) -> bool:
         len(data) > 0
         and is_list_like(data[0])
         and getattr(data[0], "ndim", 1) == 1
-        # TODO(wayd): hack so pyarrow list elements don't expand
-        and not isinstance(data[0], pa.ListScalar)
         and not isinstance(dtype, ListDtype)
         and not (isinstance(data, ExtensionArray) and data.ndim == 2)
     )

pandas/core/internals/managers.py

@@ -1976,10 +1976,7 @@ def from_blocks(
 
     @classmethod
     def from_array(
-        cls,
-        array: ArrayLike,
-        index: Index,
-        refs: BlockValuesRefs | None = None,
+        cls, array: ArrayLike, index: Index, refs: BlockValuesRefs | None = None
     ) -> SingleBlockManager:
         """
         Constructor for if we have an array that is not yet a Block.

pandas/tests/extension/list/test_list.py

@@ -10,36 +10,7 @@
     ListArray,
     ListDtype,
 )
-from pandas.tests.extension.base.accumulate import BaseAccumulateTests
-from pandas.tests.extension.base.casting import BaseCastingTests
-from pandas.tests.extension.base.constructors import BaseConstructorsTests
-from pandas.tests.extension.base.dim2 import (  # noqa: F401
-    Dim2CompatTests,
-    NDArrayBacked2DTests,
-)
-from pandas.tests.extension.base.dtype import BaseDtypeTests
-from pandas.tests.extension.base.getitem import BaseGetitemTests
-from pandas.tests.extension.base.groupby import BaseGroupbyTests
-from pandas.tests.extension.base.index import BaseIndexTests
-from pandas.tests.extension.base.interface import BaseInterfaceTests
-from pandas.tests.extension.base.io import BaseParsingTests
-from pandas.tests.extension.base.methods import BaseMethodsTests
-from pandas.tests.extension.base.missing import BaseMissingTests
-from pandas.tests.extension.base.ops import (  # noqa: F401
-    BaseArithmeticOpsTests,
-    BaseComparisonOpsTests,
-    BaseOpsUtil,
-    BaseUnaryOpsTests,
-)
-from pandas.tests.extension.base.printing import BasePrintingTests
-from pandas.tests.extension.base.reduce import BaseReduceTests
-from pandas.tests.extension.base.reshaping import BaseReshapingTests
-from pandas.tests.extension.base.setitem import BaseSetitemTests
-
-# TODO(wayd): This is copied from string tests - is it required here?
-# @pytest.fixture(params=[True, False])
-# def chunked(request):
-#     return request.param
+from pandas.tests.extension import base
 
 
 @pytest.fixture
@@ -93,27 +64,7 @@ def data_for_grouping(dtype):
     return ListArray([B, B, NA, NA, A, A, B, C])
 
 
-class TestListArray(
-    BaseAccumulateTests,
-    BaseCastingTests,
-    BaseConstructorsTests,
-    BaseDtypeTests,
-    BaseGetitemTests,
-    BaseGroupbyTests,
-    BaseIndexTests,
-    BaseInterfaceTests,
-    BaseParsingTests,
-    BaseMethodsTests,
-    BaseMissingTests,
-    BaseArithmeticOpsTests,
-    BaseComparisonOpsTests,
-    BaseUnaryOpsTests,
-    BasePrintingTests,
-    BaseReduceTests,
-    BaseReshapingTests,
-    BaseSetitemTests,
-    Dim2CompatTests,
-):
+class TestListArray(base.ExtensionTests):
     def test_fillna_no_op_returns_copy(self, data):
         # TODO(wayd): This test is copied from test_arrow.py
         # It appears the TestArrowArray class has different expectations around