Merge remote-tracking branch 'upstream/master' into STY-repr-batch-3

Author: MomIsBestFriend

asv_bench/benchmarks/boolean.py

@@ -0,0 +1,32 @@
+import numpy as np
+
+import pandas as pd
+
+
+class TimeLogicalOps:
+    def setup(self):
+        N = 10_000
+        left, right, lmask, rmask = np.random.randint(0, 2, size=(4, N)).astype("bool")
+        self.left = pd.arrays.BooleanArray(left, lmask)
+        self.right = pd.arrays.BooleanArray(right, rmask)
+
+    def time_or_scalar(self):
+        self.left | True
+        self.left | False
+
+    def time_or_array(self):
+        self.left | self.right
+
+    def time_and_scalar(self):
+        self.left & True
+        self.left & False
+
+    def time_and_array(self):
+        self.left & self.right
+
+    def time_xor_scalar(self):
+        self.left ^ True
+        self.left ^ False
+
+    def time_xor_array(self):
+        self.left ^ self.right

doc/source/index.rst.template

@@ -73,6 +73,7 @@ See the :ref:`overview` for more detail about what's in the library.
   * :doc:`user_guide/missing_data`
   * :doc:`user_guide/categorical`
   * :doc:`user_guide/integer_na`
+  * :doc:`user_guide/boolean`
   * :doc:`user_guide/visualization`
   * :doc:`user_guide/computation`
   * :doc:`user_guide/groupby`

doc/source/user_guide/boolean.rst

@@ -0,0 +1,79 @@
+.. currentmodule:: pandas
+
+.. ipython:: python
+   :suppress:
+
+   import pandas as pd
+   import numpy as np
+
+.. _boolean:
+
+**************************
+Nullable Boolean Data Type
+**************************
+
+.. versionadded:: 1.0.0
+
+.. _boolean.kleene:
+
+Kleene Logical Operations
+-------------------------
+
+:class:`arrays.BooleanArray` implements `Kleene Logic`_ (sometimes called three-value logic) for
+logical operations like ``&`` (and), ``|`` (or) and ``^`` (exclusive-or).
+
+This table demonstrates the results for every combination. These operations are symmetrical,
+so flipping the left- and right-hand side makes no difference in the result.
+
+================= =========
+Expression        Result
+================= =========
+``True & True``   ``True``
+``True & False``  ``False``
+``True & NA``     ``NA``
+``False & False`` ``False``
+``False & NA``    ``False``
+``NA & NA``       ``NA``
+``True | True``   ``True``
+``True | False``  ``True``
+``True | NA``     ``True``
+``False | False`` ``False``
+``False | NA``    ``NA``
+``NA | NA``       ``NA``
+``True ^ True``   ``False``
+``True ^ False``  ``True``
+``True ^ NA``     ``NA``
+``False ^ False`` ``False``
+``False ^ NA``    ``NA``
+``NA ^ NA``       ``NA``
+================= =========
+
+When an ``NA`` is present in an operation, the output value is ``NA`` only if
+the result cannot be determined solely based on the other input. For example,
+``True | NA`` is ``True``, because both ``True | True`` and ``True | False``
+are ``True``. In that case, we don't actually need to consider the value
+of the ``NA``.
+
+On the other hand, ``True & NA`` is ``NA``. The result depends on whether
+the ``NA`` really is ``True`` or ``False``, since ``True & True`` is ``True``,
+but ``True & False`` is ``False``, so we can't determine the output.
+
+
+This differs from how ``np.nan`` behaves in logical operations. Pandas treated
+``np.nan`` is *always false in the output*.
+
+In ``or``
+
+.. ipython:: python
+
+   pd.Series([True, False, np.nan], dtype="object") | True
+   pd.Series([True, False, np.nan], dtype="boolean") | True
+
+In ``and``
+
+.. ipython:: python
+
+   pd.Series([True, False, np.nan], dtype="object") & True
+   pd.Series([True, False, np.nan], dtype="boolean") & True
+
+.. _Kleene Logic: https://en.wikipedia.org/wiki/Three-valued_logic#Kleene_and_Priest_logics

doc/source/user_guide/index.rst

@@ -30,6 +30,7 @@ Further information on any specific method can be obtained in the
     missing_data
     categorical
     integer_na
+    boolean
     visualization
     computation
     groupby

doc/source/whatsnew/v1.0.0.rst

@@ -600,6 +600,7 @@ or ``matplotlib.Axes.plot``. See :ref:`plotting.formatters` for more.
 - In :func:`concat` the default value for ``sort`` has been changed from ``None`` to ``False`` (:issue:`20613`)
 - Removed previously deprecated "raise_conflict" argument from :meth:`DataFrame.update`, use "errors" instead (:issue:`23585`)
 - Removed previously deprecated keyword "n" from :meth:`DatetimeIndex.shift`, :meth:`TimedeltaIndex.shift`, :meth:`PeriodIndex.shift`, use "periods" instead (:issue:`22458`)
+- Removed previously deprecated keywords ``how``, ``fill_method``, and ``limit`` from :meth:`DataFrame.resample` (:issue:`30139`)
 - Passing an integer to :meth:`Series.fillna` or :meth:`DataFrame.fillna` with ``timedelta64[ns]`` dtype now raises ``TypeError`` (:issue:`24694`)
 - Passing multiple axes to :meth:`DataFrame.dropna` is no longer supported (:issue:`20995`)
 - Removed previously deprecated :meth:`Series.nonzero`, use `to_numpy().nonzero()` instead (:issue:`24048`)
@@ -621,6 +622,7 @@ or ``matplotlib.Axes.plot``. See :ref:`plotting.formatters` for more.
 - Changed :meth:`Timedelta.resolution` to match the behavior of the standard library ``datetime.timedelta.resolution``, for the old behavior, use :meth:`Timedelta.resolution_string` (:issue:`26839`)
 - Removed previously deprecated :attr:`Timestamp.weekday_name`, :attr:`DatetimeIndex.weekday_name`, and :attr:`Series.dt.weekday_name` (:issue:`18164`)
 - Removed previously deprecated ``errors`` argument in :meth:`Timestamp.tz_localize`, :meth:`DatetimeIndex.tz_localize`, and :meth:`Series.tz_localize` (:issue:`22644`)
+- Changed the default value for ``ordered`` in :class:`CategoricalDtype` from ``None`` to ``False`` (:issue:`26336`)
 - :meth:`Series.set_axis` and :meth:`DataFrame.set_axis` now require "labels" as the first argument and "axis" as an optional named parameter (:issue:`30089`)
 -
 

pandas/core/arrays/boolean.py

@@ -184,6 +184,9 @@ class BooleanArray(ExtensionArray, ExtensionOpsMixin):
     represented by 2 numpy arrays: a boolean array with the data and
     a boolean array with the mask (True indicating missing).
 
+    BooleanArray implements Kleene logic (sometimes called three-value
+    logic) for logical operations. See :ref:`boolean.kleene` for more.
+
     To construct an BooleanArray from generic array-like input, use
     :func:`pandas.array` specifying ``dtype="boolean"`` (see examples
     below).
@@ -283,7 +286,7 @@ def __getitem__(self, item):
 
     def _coerce_to_ndarray(self, dtype=None, na_value: "Scalar" = libmissing.NA):
         """
-        Coerce to an ndarary of object dtype or bool dtype (if force_bool=True).
+        Coerce to an ndarray of object dtype or bool dtype (if force_bool=True).
 
         Parameters
         ----------
@@ -565,33 +568,40 @@ def logical_method(self, other):
                 # Rely on pandas to unbox and dispatch to us.
                 return NotImplemented
 
+            assert op.__name__ in {"or_", "ror_", "and_", "rand_", "xor", "rxor"}
             other = lib.item_from_zerodim(other)
+            other_is_booleanarray = isinstance(other, BooleanArray)
+            other_is_scalar = lib.is_scalar(other)
             mask = None
 
-            if isinstance(other, BooleanArray):
+            if other_is_booleanarray:
                 other, mask = other._data, other._mask
             elif is_list_like(other):
                 other = np.asarray(other, dtype="bool")
                 if other.ndim > 1:
                     raise NotImplementedError(
                         "can only perform ops with 1-d structures"
                     )
-                if len(self) != len(other):
-                    raise ValueError("Lengths must match to compare")
                 other, mask = coerce_to_array(other, copy=False)
+            elif isinstance(other, np.bool_):
+                other = other.item()
+
+            if other_is_scalar and not (other is libmissing.NA or lib.is_bool(other)):
+                raise TypeError(
+                    "'other' should be pandas.NA or a bool. Got {} instead.".format(
+                        type(other).__name__
+                    )
+                )
 
-            # numpy will show a DeprecationWarning on invalid elementwise
-            # comparisons, this will raise in the future
-            with warnings.catch_warnings():
-                warnings.filterwarnings("ignore", "elementwise", FutureWarning)
-                with np.errstate(all="ignore"):
-                    result = op(self._data, other)
+            if not other_is_scalar and len(self) != len(other):
+                raise ValueError("Lengths must match to compare")
 
-            # nans propagate
-            if mask is None:
-                mask = self._mask
-            else:
-                mask = self._mask | mask
+            if op.__name__ in {"or_", "ror_"}:
+                result, mask = ops.kleene_or(self._data, other, self._mask, mask)
+            elif op.__name__ in {"and_", "rand_"}:
+                result, mask = ops.kleene_and(self._data, other, self._mask, mask)
+            elif op.__name__ in {"xor", "rxor"}:
+                result, mask = ops.kleene_xor(self._data, other, self._mask, mask)
 
             return BooleanArray(result, mask)
 

pandas/core/arrays/categorical.py

@@ -328,7 +328,7 @@ def __init__(
         # sanitize input
         if is_categorical_dtype(values):
             if dtype.categories is None:
-                dtype = CategoricalDtype(values.categories, dtype._ordered)
+                dtype = CategoricalDtype(values.categories, dtype.ordered)
         elif not isinstance(values, (ABCIndexClass, ABCSeries)):
             # sanitize_array coerces np.nan to a string under certain versions
             # of numpy
@@ -351,7 +351,7 @@ def __init__(
                 codes, categories = factorize(values, sort=True)
             except TypeError:
                 codes, categories = factorize(values, sort=False)
-                if dtype._ordered:
+                if dtype.ordered:
                     # raise, as we don't have a sortable data structure and so
                     # the user should give us one by specifying categories
                     raise TypeError(
@@ -367,7 +367,7 @@ def __init__(
                 )
 
             # we're inferring from values
-            dtype = CategoricalDtype(categories, dtype._ordered)
+            dtype = CategoricalDtype(categories, dtype.ordered)
 
         elif is_categorical_dtype(values):
             old_codes = (
@@ -437,7 +437,7 @@ def ordered(self) -> Ordered:
         """
         Whether the categories have an ordered relationship.
         """
-        return self.dtype._ordered
+        return self.dtype.ordered
 
     @property
     def dtype(self) -> CategoricalDtype:
@@ -833,7 +833,7 @@ def set_categories(self, new_categories, ordered=None, rename=False, inplace=Fal
         """
         inplace = validate_bool_kwarg(inplace, "inplace")
         if ordered is None:
-            ordered = self.dtype._ordered
+            ordered = self.dtype.ordered
         new_dtype = CategoricalDtype(new_categories, ordered=ordered)
 
         cat = self if inplace else self.copy()

pandas/core/construction.py

@@ -558,7 +558,7 @@ def _try_cast(
             # that Categorical is the only array type for 'category'.
             dtype = cast(CategoricalDtype, dtype)
             subarr = dtype.construct_array_type()(
-                arr, dtype.categories, ordered=dtype._ordered
+                arr, dtype.categories, ordered=dtype.ordered
             )
         elif is_extension_array_dtype(dtype):
             # create an extension array from its dtype