add extension tests for numpy_bool

Author: jreback

pandas/conftest.py

@@ -159,6 +159,20 @@ def all_numeric_reductions(request):
     return request.param
 
 
+_all_numeric_reductions_for_boolean = ['min', 'max', 'mean', 'prod',
+                                       'std', 'var', 'median',
+                                       'kurt', 'skew']
+
+
+@pytest.fixture(params=_all_numeric_reductions_for_boolean)
+def all_numeric_reductions_for_boolean(request):
+    """
+    Fixture for numeric reduction names that are not allowed
+    for boolean.
+    """
+    return request.param
+
+
 _all_boolean_reductions = ['all', 'any']
 
 

pandas/core/arrays/integer.py

@@ -14,7 +14,7 @@
     is_bool_dtype, is_float, is_float_dtype, is_integer, is_integer_dtype,
     is_list_like, is_object_dtype, is_scalar)
 from pandas.core.dtypes.dtypes import register_extension_dtype
-from pandas.core.dtypes.generic import ABCIndexClass, ABCMaskArray, ABCSeries
+from pandas.core.dtypes.generic import ABCIndexClass, ABCSeries
 from pandas.core.dtypes.missing import isna, notna
 
 from pandas.core import nanops
@@ -288,8 +288,7 @@ def __init__(self, values, mask, copy=False):
                 and is_integer_dtype(values.dtype)):
             raise TypeError("values should be integer numpy array. Use "
                             "the 'integer_array' function instead")
-        if not (isinstance(mask, (np.ndarray, ABCMaskArray)) and
-                is_bool_dtype(mask.dtype)):
+        if not is_bool_dtype(mask):
             raise TypeError("mask should be boolean numpy array. Use "
                             "the 'integer_array' function instead")
 

pandas/core/arrays/mask/_base.py

@@ -1,4 +1,4 @@
-"""A boolean mask interace.
+"""A boolean mask intferace.
 
 This module provides an interface to a numpy / pyarrow boolean mask.
 This is limited as not all of the implementations can hold NA, so
@@ -11,6 +11,7 @@
 
 from pandas import compat
 from pandas.api.extensions import ExtensionDtype
+from pandas.api.types import is_scalar
 from pandas.core.arrays.base import ExtensionArray
 from pandas.core.missing import isna
 
@@ -39,7 +40,7 @@ def __eq__(self, other):
         # compare == to np.dtype('bool')
         if isinstance(other, compat.string_types):
             return other == self.name
-        elif other is self:
+        elif isinstance(other, type(self)):
             return True
         elif isinstance(other, np.dtype):
             return other == 'bool'
@@ -55,10 +56,6 @@ class BoolArray(ExtensionArray):
     def _from_sequence(cls, scalars, dtype=None, copy=False):
         return cls.from_scalars(scalars)
 
-    @property
-    def dtype(self):
-        return self._dtype
-
     @property
     def size(self):
         return len(self)
@@ -94,13 +91,21 @@ def __iand__(self, other):
         return type(self).from_scalars(
             np.array(self, copy=False) & (np.array(other, copy=False)))
 
+    def __getitem__(self, item):
+        arr = np.array(self, copy=False)
+        if is_scalar(item):
+            return arr[item]
+        else:
+            arr = arr[item]
+            return type(self).from_scalars(arr)
+
     def view(self, dtype=None):
         arr = np.array(self._data, copy=False)
         if dtype is not None:
             arr = arr.view(dtype=dtype)
         return arr
 
-    def sum(self, axis=None):
+    def sum(self, axis=None, min_count=None):
         return np.array(self, copy=False).sum()
 
     def copy(self, deep=False):
@@ -120,9 +125,11 @@ def _reduce(self, method, skipna=True, **kwargs):
             arr = self[~self.isna()]
         else:
             arr = self
-
+        # we only allow explicity defined methods
+        # ndarrays actually support: mean, var, prod, min, max
         try:
             op = getattr(arr, method)
+            return op()
         except AttributeError:
-            raise TypeError
-        return op(**kwargs)
+            pass
+        raise TypeError

pandas/core/arrays/mask/_numpy.py

@@ -21,6 +21,8 @@ class NumpyBoolArray(BoolArray):
     """Generic class which can be used to represent missing data.
     """
 
+    dtype = NumpyBoolDtype()
+
     @classmethod
     def from_scalars(cls, values):
         arr = np.asarray(values).astype(np.bool_, copy=False)
@@ -39,10 +41,6 @@ def __init__(self, mask, copy=True):
         if copy:
             mask = mask.copy()
         self._data = mask
-        self._dtype = NumpyBoolDtype()
-
-    def __getitem__(self, key):
-        return self._data[key]
 
     def __setitem__(self, key, value):
         self._data[key] = value
@@ -60,8 +58,13 @@ def nbytes(self):
     def reshape(self, shape, **kwargs):
         return np.array(self, copy=False).reshape(shape, **kwargs)
 
-    def astype(self, dtype, copy=False):
-        return np.array(self, copy=False).astype(dtype, copy=copy)
+    def astype(self, dtype, copy=True):
+        # needed to fix this astype for the Series constructor.
+        if isinstance(dtype, type(self.dtype)) and dtype == self.dtype:
+            if copy:
+                return self.copy()
+            return self
+        return super(NumpyBoolArray, self).astype(dtype, copy)
 
     def take(self, indices, allow_fill=False, fill_value=None, axis=None):
         # TODO: had to add axis here
@@ -73,3 +76,7 @@ def take(self, indices, allow_fill=False, fill_value=None, axis=None):
         result = take(data, indices, fill_value=fill_value,
                       allow_fill=allow_fill)
         return self._from_sequence(result, dtype=self.dtype)
+
+    def _concat_same_type(cls, to_concat):
+        concat = np.concatenate(to_concat)
+        return cls.from_scalars(concat)

pandas/core/arrays/mask/_pyarrow.py

@@ -11,7 +11,6 @@
 import numpy as np
 
 from pandas.api.extensions import take
-from pandas.api.types import is_scalar
 from pandas.core.arrays.mask._base import BoolArray, BoolDtype
 
 # we require pyarrow >= 0.10.0
@@ -35,6 +34,8 @@ def construct_array_type(cls):
 
 class ArrowBoolArray(BoolArray):
 
+    dtype = ArrowBoolDtype()
+
     @classmethod
     def from_scalars(cls, values):
         values = np.asarray(values).astype(np.bool_, copy=False)
@@ -53,14 +54,6 @@ def __init__(self, values, copy=False):
             values = values.copy()
 
         self._data = values
-        self._dtype = ArrowBoolDtype()
-
-    def __getitem__(self, item):
-        if is_scalar(item):
-            return self._data.to_pandas()[item]
-        else:
-            vals = self._data.to_pandas()[item]
-            return type(self).from_scalars(vals)
 
     def __setitem__(self, key, value):
         # TODO: hack-a-minute

pandas/tests/arrays/mask/test_mask.py

@@ -6,21 +6,29 @@
 
 
 @pytest.fixture(params=['numpy', 'arrow', 'mask'])
-def mask_type(request):
+def mask_dtype(request):
+    """ dtype type """
     if request.param == 'numpy':
-        from pandas.core.arrays.mask._numpy import NumpyBoolArray
-        return NumpyBoolArray
+        from pandas.core.arrays.mask._numpy import NumpyBoolDtype
+        return NumpyBoolDtype
     elif request.param == 'arrow':
         pytest.importorskip('pyarrow', minversion="0.10.0")
-        from pandas.core.arrays.mask._pyarrow import ArrowBoolArray
-        return ArrowBoolArray
+        from pandas.core.arrays.mask._pyarrow import ArrowBoolDtype
+        return ArrowBoolDtype
     elif request.param == 'mask':
         from pandas.core.arrays.mask import get_mask_array_type
-        return get_mask_array_type()
+        return type(get_mask_array_type().dtype)
+
+
+@pytest.fixture
+def mask_type(mask_dtype):
+    """ array type """
+    return mask_dtype.construct_array_type()
 
 
 @pytest.fixture
 def mask(mask_type):
+    """ array object """
     return mask_type._from_sequence([1, 0, 1])
 
 
@@ -93,3 +101,10 @@ def test_functions(mask):
     assert (mask2 == mask).all()
 
     assert mask.size == len(mask)
+
+
+def test_dtype(mask_dtype):
+    m = mask_dtype()
+    assert m == m
+    assert m == mask_dtype()
+    assert hash(m) is not None

pandas/tests/extension/mask/__init__.py

@@ -0,0 +1,80 @@
+"""
+This file contains a minimal set of tests for compliance with the extension
+array interface test suite, and should contain no other tests.
+The test suite for the full functionality of the array is located in
+`pandas/tests/arrays/`.
+
+The tests in this file are inherited from the BaseExtensionTests, and only
+minimal tweaks should be applied to get the tests passing (by overwriting a
+parent method).
+
+Additional tests should either be added to one of the BaseExtensionTests
+classes (if they are relevant for the extension interface for all dtypes), or
+be added to the array-specific tests in `pandas/tests/arrays/`.
+
+"""
+import numpy as np
+import pytest
+
+import pandas as pd
+from pandas.tests.extension import base
+import pandas.util.testing as tm
+
+from pandas.core.arrays.mask._numpy import (
+    NumpyBoolArray, NumpyBoolDtype)
+
+
+@pytest.fixture
+def dtype():
+    return NumpyBoolDtype()
+
+
+@pytest.fixture
+def data():
+    return NumpyBoolArray.from_scalars(np.random.randint(0, 2, size=100,
+                                                         dtype=bool))
+
+
+@pytest.fixture
+def data_missing():
+    pytest.skip("not supported in NumpyBoolArray")
+
+
+class BaseNumpyTests(object):
+    pass
+
+
+class TestDtype(BaseNumpyTests, base.BaseDtypeTests):
+    def test_array_type_with_arg(self, data, dtype):
+        pytest.skip("GH-22666")
+
+
+class TestInterface(BaseNumpyTests, base.BaseInterfaceTests):
+    def test_repr(self, data):
+        raise pytest.skip("TODO")
+
+
+class TestConstructors(BaseNumpyTests, base.BaseConstructorsTests):
+    def test_from_dtype(self, data):
+        pytest.skip("GH-22666")
+
+
+class TestReduceBoolean(base.BaseBooleanReduceTests):
+
+    @pytest.mark.parametrize('skipna', [True, False])
+    def test_reduce_series_numeric(
+            self, data, all_numeric_reductions_for_boolean, skipna):
+        op_name = all_numeric_reductions_for_boolean
+        s = pd.Series(data)
+
+        with pytest.raises(TypeError):
+            getattr(s, op_name)(skipna=skipna)
+
+
+def test_is_bool_dtype(data):
+    assert pd.api.types.is_bool_dtype(data)
+    assert pd.core.common.is_bool_indexer(data)
+    s = pd.Series(range(len(data)))
+    result = s[data]
+    expected = s[np.asarray(data)]
+    tm.assert_series_equal(result, expected)

pandas/tests/extension/mask/test_numpy_bool.py

@@ -1,3 +1,18 @@
+"""
+This file contains a minimal set of tests for compliance with the extension
+array interface test suite, and should contain no other tests.
+The test suite for the full functionality of the array is located in
+`pandas/tests/arrays/`.
+
+The tests in this file are inherited from the BaseExtensionTests, and only
+minimal tweaks should be applied to get the tests passing (by overwriting a
+parent method).
+
+Additional tests should either be added to one of the BaseExtensionTests
+classes (if they are relevant for the extension interface for all dtypes), or
+be added to the array-specific tests in `pandas/tests/arrays/`.
+
+"""
 import numpy as np
 import pytest
 
@@ -46,13 +61,16 @@ def test_from_dtype(self, data):
         pytest.skip("GH-22666")
 
 
-class TestReduce(base.BaseNoReduceTests):
-    def test_reduce_series_boolean(self):
-        pass
+class TestReduceBoolean(base.BaseBooleanReduceTests):
 
+    @pytest.mark.parametrize('skipna', [True, False])
+    def test_reduce_series_numeric(
+            self, data, all_numeric_reductions_for_boolean, skipna):
+        op_name = all_numeric_reductions_for_boolean
+        s = pd.Series(data)
 
-class TestReduceBoolean(base.BaseBooleanReduceTests):
-    pass
+        with pytest.raises(TypeError):
+            getattr(s, op_name)(skipna=skipna)
 
 
 def test_is_bool_dtype(data):