Merge remote-tracking branch 'upstream/master' into bisect

Author: simonjayhawkins

.travis.yml

@@ -35,12 +35,6 @@ matrix:
   fast_finish: true
 
   include:
-    - env:
-      - JOB="3.8, slow" ENV_FILE="ci/deps/travis-38-slow.yaml" PATTERN="slow" SQL="1"
-      services:
-        - mysql
-        - postgresql
-
     - env:
         - JOB="3.7, locale" ENV_FILE="ci/deps/travis-37-locale.yaml" PATTERN="((not slow and not network and not clipboard) or (single and db))" LOCALE_OVERRIDE="zh_CN.UTF-8" SQL="1"
       services:

ci/azure/posix.yml

@@ -43,6 +43,11 @@ jobs:
           CONDA_PY: "38"
           PATTERN: "not slow and not network and not clipboard"
 
+        py38_slow:
+          ENV_FILE: ci/deps/azure-38-slow.yaml
+          CONDA_PY: "38"
+          PATTERN: "slow"
+
         py38_locale:
           ENV_FILE: ci/deps/azure-38-locale.yaml
           CONDA_PY: "38"

ci/deps/travis-38-slow.yaml renamed to ci/deps/azure-38-slow.yaml

@@ -474,15 +474,16 @@ A directory of projects providing
 :ref:`extension accessors <extending.register-accessors>`. This is for users to
 discover new accessors and for library authors to coordinate on the namespace.
 
-=============== ========== ========================= ===============================================================
-Library         Accessor   Classes                   Description
-=============== ========== ========================= ===============================================================
-`cyberpandas`_  ``ip``     ``Series``                Provides common operations for working with IP addresses.
-`pdvega`_       ``vgplot`` ``Series``, ``DataFrame`` Provides plotting functions from the Altair_ library.
-`pandas_path`_  ``path``   ``Index``, ``Series``     Provides `pathlib.Path`_ functions for Series.
-`pint-pandas`_  ``pint``   ``Series``, ``DataFrame`` Provides units support for numeric Series and DataFrames.
-`composeml`_    ``slice``  ``DataFrame``             Provides a generator for enhanced data slicing.
-=============== ========== ========================= ===============================================================
+=============== ============ ==================================== ===============================================================
+Library         Accessor     Classes                              Description
+=============== ============ ==================================== ===============================================================
+`cyberpandas`_  ``ip``       ``Series``                           Provides common operations for working with IP addresses.
+`pdvega`_       ``vgplot``   ``Series``, ``DataFrame``            Provides plotting functions from the Altair_ library.
+`pandas_path`_  ``path``     ``Index``, ``Series``                Provides `pathlib.Path`_ functions for Series.
+`pint-pandas`_  ``pint``     ``Series``, ``DataFrame``            Provides units support for numeric Series and DataFrames.
+`composeml`_    ``slice``    ``DataFrame``                        Provides a generator for enhanced data slicing.
+`datatest`_     ``validate`` ``Series``, ``DataFrame``, ``Index`` Provides validation, differences, and acceptance managers.
+=============== ============ ==================================== ===============================================================
 
 .. _cyberpandas: https://cyberpandas.readthedocs.io/en/latest
 .. _pdvega: https://altair-viz.github.io/pdvega/
@@ -492,3 +493,4 @@ Library         Accessor   Classes                   Description
 .. _pathlib.Path: https://docs.python.org/3/library/pathlib.html
 .. _pint-pandas: https://github.com/hgrecco/pint-pandas
 .. _composeml: https://github.com/FeatureLabs/compose
+.. _datatest: https://datatest.readthedocs.io/

doc/source/ecosystem.rst

@@ -747,6 +747,7 @@ I/O
 - :meth:`DataFrame.to_html` was ignoring ``formatters`` argument for ``ExtensionDtype`` columns (:issue:`36525`)
 - Bumped minimum xarray version to 0.12.3 to avoid reference to the removed ``Panel`` class (:issue:`27101`)
 - :meth:`DataFrame.to_csv` was re-opening file-like handles that also implement ``os.PathLike`` (:issue:`38125`)
+- Bug in the conversion of a sliced ``pyarrow.Table`` with missing values to a DataFrame (:issue:`38525`)
 
 Period
 ^^^^^^
@@ -859,7 +860,7 @@ Other
 - Bug in :meth:`RangeIndex.difference` returning :class:`Int64Index` in some cases where it should return :class:`RangeIndex` (:issue:`38028`)
 - Fixed bug in :func:`assert_series_equal` when comparing a datetime-like array with an equivalent non extension dtype array (:issue:`37609`)
 - Bug in :func:`.is_bool_dtype` would raise when passed a valid string such as ``"boolean"`` (:issue:`38386`)
-
+- Fixed regression in logical operators raising ``ValueError`` when columns of :class:`DataFrame` are a :class:`CategoricalIndex` with unused categories (:issue:`38367`)
 
 .. ---------------------------------------------------------------------------
 

doc/source/whatsnew/v1.2.0.rst

@@ -30,7 +30,7 @@ def pyarrow_array_to_numpy_and_mask(arr, dtype):
     bitmask = buflist[0]
     if bitmask is not None:
         mask = pyarrow.BooleanArray.from_buffers(
-            pyarrow.bool_(), len(arr), [None, bitmask]
+            pyarrow.bool_(), len(arr), [None, bitmask], offset=arr.offset
         )
         mask = np.asarray(mask)
     else:

pandas/core/arrays/_arrow_utils.py

@@ -10,15 +10,14 @@
 
 from pandas.core.dtypes.common import (
     is_bool_dtype,
-    is_extension_array_dtype,
     is_float,
     is_float_dtype,
     is_integer_dtype,
     is_list_like,
     is_numeric_dtype,
     pandas_dtype,
 )
-from pandas.core.dtypes.dtypes import register_extension_dtype
+from pandas.core.dtypes.dtypes import ExtensionDtype, register_extension_dtype
 from pandas.core.dtypes.missing import isna
 
 from pandas.core import ops
@@ -372,34 +371,22 @@ def astype(self, dtype, copy: bool = True) -> ArrayLike:
             if incompatible type with an BooleanDtype, equivalent of same_kind
             casting
         """
-        from pandas.core.arrays.string_ import StringDtype
-
         dtype = pandas_dtype(dtype)
 
-        if isinstance(dtype, BooleanDtype):
-            values, mask = coerce_to_array(self, copy=copy)
-            if not copy:
-                return self
-            else:
-                return BooleanArray(values, mask, copy=False)
-        elif isinstance(dtype, StringDtype):
-            return dtype.construct_array_type()._from_sequence(self, copy=False)
+        if isinstance(dtype, ExtensionDtype):
+            return super().astype(dtype, copy)
 
         if is_bool_dtype(dtype):
             # astype_nansafe converts np.nan to True
             if self._hasna:
                 raise ValueError("cannot convert float NaN to bool")
             else:
                 return self._data.astype(dtype, copy=copy)
-        if is_extension_array_dtype(dtype) and is_integer_dtype(dtype):
-            from pandas.core.arrays import IntegerArray
 
-            return IntegerArray(
-                self._data.astype(dtype.numpy_dtype), self._mask.copy(), copy=False
-            )
         # for integer, error if there are missing values
         if is_integer_dtype(dtype) and self._hasna:
             raise ValueError("cannot convert NA to integer")
+
         # for float dtype, ensure we use np.nan before casting (numpy cannot
         # deal with pd.NA)
         na_value = self._na_value

pandas/core/arrays/boolean.py

@@ -346,7 +346,7 @@ def astype(self, dtype, copy=True):
         elif is_string_dtype(dtype) and not is_categorical_dtype(dtype):
             if is_extension_array_dtype(dtype):
                 arr_cls = dtype.construct_array_type()
-                return arr_cls._from_sequence(self, dtype=dtype)
+                return arr_cls._from_sequence(self, dtype=dtype, copy=copy)
             else:
                 return self._format_native_types()
         elif is_integer_dtype(dtype):

pandas/core/arrays/datetimelike.py

@@ -31,7 +31,6 @@
     is_categorical_dtype,
     is_datetime64_any_dtype,
     is_datetime64_dtype,
-    is_datetime64_ns_dtype,
     is_datetime64tz_dtype,
     is_dtype_equal,
     is_extension_array_dtype,
@@ -587,24 +586,30 @@ def astype(self, dtype, copy=True):
         # DatetimeLikeArrayMixin Super handles the rest.
         dtype = pandas_dtype(dtype)
 
-        if is_datetime64_ns_dtype(dtype) and not is_dtype_equal(dtype, self.dtype):
+        if is_dtype_equal(dtype, self.dtype):
+            if copy:
+                return self.copy()
+            return self
+
+        elif is_datetime64tz_dtype(dtype) and self.tz is None:
+            # FIXME: GH#33401 this does not match Series behavior
+            return self.tz_localize(dtype.tz)
+
+        elif is_datetime64tz_dtype(dtype):
             # GH#18951: datetime64_ns dtype but not equal means different tz
-            new_tz = getattr(dtype, "tz", None)
-            if getattr(self.dtype, "tz", None) is None:
-                return self.tz_localize(new_tz)
-            result = self.tz_convert(new_tz)
+            result = self.tz_convert(dtype.tz)
             if copy:
                 result = result.copy()
-            if new_tz is None:
-                # Do we want .astype('datetime64[ns]') to be an ndarray.
-                # The astype in Block._astype expects this to return an
-                # ndarray, but we could maybe work around it there.
-                result = result._data
             return result
-        elif is_datetime64tz_dtype(self.dtype) and is_dtype_equal(self.dtype, dtype):
+
+        elif dtype == "M8[ns]":
+            # we must have self.tz is None, otherwise we would have gone through
+            #  the is_dtype_equal branch above.
+            result = self.tz_convert("UTC").tz_localize(None)
             if copy:
-                return self.copy()
-            return self
+                result = result.copy()
+            return result
+
         elif is_period_dtype(dtype):
             return self.to_period(freq=dtype.freq)
         return dtl.DatetimeLikeArrayMixin.astype(self, dtype, copy)

pandas/core/arrays/datetimes.py

@@ -19,14 +19,13 @@
     is_object_dtype,
     pandas_dtype,
 )
-from pandas.core.dtypes.dtypes import register_extension_dtype
+from pandas.core.dtypes.dtypes import ExtensionDtype, register_extension_dtype
 from pandas.core.dtypes.missing import isna
 
 from pandas.core import ops
 from pandas.core.ops import invalid_comparison
 from pandas.core.tools.numeric import to_numeric
 
-from .masked import BaseMaskedDtype
 from .numeric import NumericArray, NumericDtype
 
 
@@ -332,24 +331,10 @@ def astype(self, dtype, copy: bool = True) -> ArrayLike:
             if incompatible type with an FloatingDtype, equivalent of same_kind
             casting
         """
-        from pandas.core.arrays.string_ import StringArray, StringDtype
-
         dtype = pandas_dtype(dtype)
 
-        # if the dtype is exactly the same, we can fastpath
-        if self.dtype == dtype:
-            # return the same object for copy=False
-            return self.copy() if copy else self
-        # if we are astyping to another nullable masked dtype, we can fastpath
-        if isinstance(dtype, BaseMaskedDtype):
-            # TODO deal with NaNs
-            data = self._data.astype(dtype.numpy_dtype, copy=copy)
-            # mask is copied depending on whether the data was copied, and
-            # not directly depending on the `copy` keyword
-            mask = self._mask if data is self._data else self._mask.copy()
-            return dtype.construct_array_type()(data, mask, copy=False)
-        elif isinstance(dtype, StringDtype):
-            return StringArray._from_sequence(self, copy=False)
+        if isinstance(dtype, ExtensionDtype):
+            return super().astype(dtype, copy=copy)
 
         # coerce
         if is_float_dtype(dtype):

pandas/core/arrays/floating.py

@@ -9,7 +9,7 @@
 from pandas.compat.numpy import function as nv
 from pandas.util._decorators import cache_readonly
 
-from pandas.core.dtypes.base import register_extension_dtype
+from pandas.core.dtypes.base import ExtensionDtype, register_extension_dtype
 from pandas.core.dtypes.common import (
     is_bool_dtype,
     is_datetime64_dtype,
@@ -390,24 +390,10 @@ def astype(self, dtype, copy: bool = True) -> ArrayLike:
             if incompatible type with an IntegerDtype, equivalent of same_kind
             casting
         """
-        from pandas.core.arrays.masked import BaseMaskedDtype
-        from pandas.core.arrays.string_ import StringDtype
-
         dtype = pandas_dtype(dtype)
 
-        # if the dtype is exactly the same, we can fastpath
-        if self.dtype == dtype:
-            # return the same object for copy=False
-            return self.copy() if copy else self
-        # if we are astyping to another nullable masked dtype, we can fastpath
-        if isinstance(dtype, BaseMaskedDtype):
-            data = self._data.astype(dtype.numpy_dtype, copy=copy)
-            # mask is copied depending on whether the data was copied, and
-            # not directly depending on the `copy` keyword
-            mask = self._mask if data is self._data else self._mask.copy()
-            return dtype.construct_array_type()(data, mask, copy=False)
-        elif isinstance(dtype, StringDtype):
-            return dtype.construct_array_type()._from_sequence(self, copy=False)
+        if isinstance(dtype, ExtensionDtype):
+            return super().astype(dtype, copy=copy)
 
         # coerce
         if is_float_dtype(dtype):

pandas/core/arrays/integer.py

@@ -5,16 +5,18 @@
 import numpy as np
 
 from pandas._libs import lib, missing as libmissing
-from pandas._typing import Scalar
+from pandas._typing import ArrayLike, Dtype, Scalar
 from pandas.errors import AbstractMethodError
 from pandas.util._decorators import cache_readonly, doc
 
 from pandas.core.dtypes.base import ExtensionDtype
 from pandas.core.dtypes.common import (
+    is_dtype_equal,
     is_integer,
     is_object_dtype,
     is_scalar,
     is_string_dtype,
+    pandas_dtype,
 )
 from pandas.core.dtypes.missing import isna, notna
 
@@ -229,6 +231,30 @@ def to_numpy(
             data = self._data.astype(dtype, copy=copy)
         return data
 
+    def astype(self, dtype: Dtype, copy: bool = True) -> ArrayLike:
+        dtype = pandas_dtype(dtype)
+
+        if is_dtype_equal(dtype, self.dtype):
+            if copy:
+                return self.copy()
+            return self
+
+        # if we are astyping to another nullable masked dtype, we can fastpath
+        if isinstance(dtype, BaseMaskedDtype):
+            # TODO deal with NaNs for FloatingArray case
+            data = self._data.astype(dtype.numpy_dtype, copy=copy)
+            # mask is copied depending on whether the data was copied, and
+            # not directly depending on the `copy` keyword
+            mask = self._mask if data is self._data else self._mask.copy()
+            cls = dtype.construct_array_type()
+            return cls(data, mask, copy=False)
+
+        if isinstance(dtype, ExtensionDtype):
+            eacls = dtype.construct_array_type()
+            return eacls._from_sequence(self, dtype=dtype, copy=copy)
+
+        raise NotImplementedError("subclass must implement astype to np.dtype")
+
     __array_priority__ = 1000  # higher than ndarray so ops dispatch to us
 
     def __array__(self, dtype=None) -> np.ndarray:

pandas/core/arrays/masked.py

@@ -27,7 +27,7 @@
     period_asfreq_arr,
 )
 from pandas._typing import AnyArrayLike
-from pandas.util._decorators import cache_readonly
+from pandas.util._decorators import cache_readonly, doc
 
 from pandas.core.dtypes.common import (
     TD64NS_DTYPE,
@@ -51,6 +51,10 @@
 from pandas.core.arrays import datetimelike as dtl
 import pandas.core.common as com
 
+_shared_doc_kwargs = {
+    "klass": "PeriodArray",
+}
+
 
 def _field_accessor(name: str, docstring=None):
     def f(self):
@@ -67,8 +71,8 @@ class PeriodArray(PeriodMixin, dtl.DatelikeOps):
     """
     Pandas ExtensionArray for storing Period data.
 
-    Users should use :func:`period_range` to create new instances.
-    Alternatively, :func:`array` can be used to create new instances
+    Users should use :func:`~pandas.period_array` to create new instances.
+    Alternatively, :func:`~pandas.array` can be used to create new instances
     from a sequence of Period scalars.
 
     Parameters
@@ -495,15 +499,19 @@ def _time_shift(self, periods, freq=None):
     def _box_func(self, x) -> Union[Period, NaTType]:
         return Period._from_ordinal(ordinal=x, freq=self.freq)
 
+    @doc(**_shared_doc_kwargs, other="PeriodIndex", other_name="PeriodIndex")
     def asfreq(self, freq=None, how: str = "E") -> "PeriodArray":
         """
-        Convert the Period Array/Index to the specified frequency `freq`.
+        Convert the {klass} to the specified frequency `freq`.
+
+        Equivalent to applying :meth:`pandas.Period.asfreq` with the given arguments
+        to each :class:`~pandas.Period` in this {klass}.
 
         Parameters
         ----------
         freq : str
             A frequency.
-        how : str {'E', 'S'}
+        how : str {{'E', 'S'}}, default 'E'
             Whether the elements should be aligned to the end
             or start within pa period.
 
@@ -514,8 +522,13 @@ def asfreq(self, freq=None, how: str = "E") -> "PeriodArray":
 
         Returns
         -------
-        Period Array/Index
-            Constructed with the new frequency.
+        {klass}
+            The transformed {klass} with the new frequency.
+
+        See Also
+        --------
+        {other}.asfreq: Convert each Period in a {other_name} to the given frequency.
+        Period.asfreq : Convert a :class:`~pandas.Period` object to the given frequency.
 
         Examples
         --------