Merge remote-tracking branch 'upstream/master' into typ_c_parser

Author: phofl

doc/source/whatsnew/v0.10.0.rst

@@ -181,6 +181,7 @@ labeled the aggregated group with the end of the interval: the next day).
   ``X0``, ``X1``, ...) can be reproduced by specifying ``prefix='X'``:
 
 .. ipython:: python
+   :okwarning:
 
     import io
 
@@ -197,11 +198,25 @@ labeled the aggregated group with the end of the interval: the next day).
   though this can be controlled by new ``true_values`` and ``false_values``
   arguments:
 
-.. ipython:: python
+.. code-block:: ipython
 
-    print(data)
-    pd.read_csv(io.StringIO(data))
-    pd.read_csv(io.StringIO(data), true_values=["Yes"], false_values=["No"])
+    In [4]: print(data)
+
+        a,b,c
+        1,Yes,2
+        3,No,4
+
+    In [5]: pd.read_csv(io.StringIO(data))
+    Out[5]:
+           a    b  c
+    0      1  Yes  2
+    1      3   No  4
+
+    In [6]: pd.read_csv(io.StringIO(data), true_values=["Yes"], false_values=["No"])
+    Out[6]:
+           a      b  c
+    0      1   True  2
+    1      3  False  4
 
 - The file parsers will not recognize non-string values arising from a
   converter function as NA if passed in the ``na_values`` argument. It's better

doc/source/whatsnew/v1.4.0.rst

@@ -164,6 +164,14 @@ Previously, negative arguments returned empty frames.
     df.groupby("A").nth(slice(1, -1))
     df.groupby("A").nth([slice(None, 1), slice(-1, None)])
 
+:meth:`.GroupBy.nth` now accepts index notation.
+
+.. ipython:: python
+
+    df.groupby("A").nth[1, -1]
+    df.groupby("A").nth[1:-1]
+    df.groupby("A").nth[:1, -1:]
+
 .. _whatsnew_140.dict_tight:
 
 DataFrame.from_dict and DataFrame.to_dict have new ``'tight'`` option
@@ -804,6 +812,8 @@ ExtensionArray
 - Avoid raising ``PerformanceWarning`` about fragmented DataFrame when using many columns with an extension dtype (:issue:`44098`)
 - Bug in :class:`IntegerArray` and :class:`FloatingArray` construction incorrectly coercing mismatched NA values (e.g. ``np.timedelta64("NaT")``) to numeric NA (:issue:`44514`)
 - Bug in :meth:`BooleanArray.__eq__` and :meth:`BooleanArray.__ne__` raising ``TypeError`` on comparison with an incompatible type (like a string). This caused :meth:`DataFrame.replace` to sometimes raise a ``TypeError`` if a nullable boolean column was included (:issue:`44499`)
+- Bug in :func:`array` incorrectly raising when passed a ``ndarray`` with ``float16`` dtype (:issue:`44715`)
+- Bug in calling ``np.sqrt`` on :class:`BooleanArray` returning a malformed :class:`FloatingArray` (:issue:`44715`)
 - Bug in :meth:`Series.where` with ``ExtensionDtype`` when ``other`` is a NA scalar incompatible with the series dtype (e.g. ``NaT`` with a numeric dtype) incorrectly casting to a compatible NA value (:issue:`44697`)
 -
 

pandas/core/arrays/_mixins.py

@@ -339,14 +339,6 @@ def fillna(
     # ------------------------------------------------------------------------
     # Reductions
 
-    def _reduce(self, name: str, *, skipna: bool = True, **kwargs):
-        meth = getattr(self, name, None)
-        if meth:
-            return meth(skipna=skipna, **kwargs)
-        else:
-            msg = f"'{type(self).__name__}' does not implement reduction '{name}'"
-            raise TypeError(msg)
-
     def _wrap_reduction_result(self, axis: int | None, result):
         if axis is None or self.ndim == 1:
             return self._box_func(result)

pandas/core/arrays/base.py

@@ -1352,7 +1352,13 @@ def _reduce(self, name: str, *, skipna: bool = True, **kwargs):
         ------
         TypeError : subclass does not define reductions
         """
-        raise TypeError(f"cannot perform {name} with type {self.dtype}")
+        meth = getattr(self, name, None)
+        if meth is None:
+            raise TypeError(
+                f"'{type(self).__name__}' with dtype {self.dtype} "
+                f"does not support reduction '{name}'"
+            )
+        return meth(skipna=skipna, **kwargs)
 
     # https://github.com/python/typeshed/issues/2148#issuecomment-520783318
     # Incompatible types in assignment (expression has type "None", base class

pandas/core/arrays/categorical.py

@@ -940,7 +940,7 @@ def set_categories(
                 "a future version. Removing unused categories will always "
                 "return a new Categorical object.",
                 FutureWarning,
-                stacklevel=find_stack_level(),
+                stacklevel=2,
             )
         else:
             inplace = False

pandas/core/arrays/floating.py

@@ -254,6 +254,10 @@ def __init__(self, values: np.ndarray, mask: np.ndarray, copy: bool = False):
                 "values should be floating numpy array. Use "
                 "the 'pd.array' function instead"
             )
+        if values.dtype == np.float16:
+            # If we don't raise here, then accessing self.dtype would raise
+            raise TypeError("FloatingArray does not support np.float16 dtype.")
+
         super().__init__(values, mask, copy=copy)
 
     @classmethod

pandas/core/arrays/masked.py

@@ -466,6 +466,10 @@ def reconstruct(x):
                 return IntegerArray(x, m)
             elif is_float_dtype(x.dtype):
                 m = mask.copy()
+                if x.dtype == np.float16:
+                    # reached in e.g. np.sqrt on BooleanArray
+                    # we don't support float16
+                    x = x.astype(np.float32)
                 return FloatingArray(x, m)
             else:
                 x[mask] = np.nan

pandas/core/arrays/sparse/array.py

@@ -1363,13 +1363,6 @@ def _reduce(self, name: str, *, skipna: bool = True, **kwargs):
         else:
             arr = self.dropna()
 
-        # we don't support these kwargs.
-        # They should only be present when called via pandas, so do it here.
-        # instead of in `any` / `all` (which will raise if they're present,
-        # thanks to nv.validate
-        kwargs.pop("filter_type", None)
-        kwargs.pop("numeric_only", None)
-        kwargs.pop("op", None)
         return getattr(arr, name)(**kwargs)
 
     def all(self, axis=None, *args, **kwargs):

pandas/core/arrays/string_arrow.py

@@ -335,12 +335,6 @@ def _as_pandas_scalar(self, arrow_scalar: pa.Scalar):
         else:
             return scalar
 
-    def _reduce(self, name: str, skipna: bool = True, **kwargs):
-        if name in ["min", "max"]:
-            return getattr(self, name)(skipna=skipna)
-
-        raise TypeError(f"Cannot perform reduction '{name}' with string dtype")
-
     @property
     def nbytes(self) -> int:
         """

pandas/core/construction.py

@@ -353,7 +353,12 @@ def array(
         elif inferred_dtype == "integer":
             return IntegerArray._from_sequence(data, copy=copy)
 
-        elif inferred_dtype in ("floating", "mixed-integer-float"):
+        elif (
+            inferred_dtype in ("floating", "mixed-integer-float")
+            and getattr(data, "dtype", None) != np.float16
+        ):
+            # GH#44715 Exclude np.float16 bc FloatingArray does not support it;
+            #  we will fall back to PandasArray.
             return FloatingArray._from_sequence(data, copy=copy)
 
         elif inferred_dtype == "boolean":

pandas/core/frame.py

@@ -9183,6 +9183,29 @@ def join(
         3  K3  A3  NaN
         4  K4  A4  NaN
         5  K5  A5  NaN
+
+        Using non-unique key values shows how they are matched.
+
+        >>> df = pd.DataFrame({'key': ['K0', 'K1', 'K1', 'K3', 'K0', 'K1'],
+        ...                    'A': ['A0', 'A1', 'A2', 'A3', 'A4', 'A5']})
+
+        >>> df
+          key   A
+        0  K0  A0
+        1  K1  A1
+        2  K1  A2
+        3  K3  A3
+        4  K0  A4
+        5  K1  A5
+
+        >>> df.join(other.set_index('key'), on='key')
+          key   A    B
+        0  K0  A0   B0
+        1  K1  A1   B1
+        2  K1  A2   B1
+        3  K3  A3  NaN
+        4  K0  A4   B0
+        5  K1  A5   B1
         """
         return self._join_compat(
             other, on=on, how=how, lsuffix=lsuffix, rsuffix=rsuffix, sort=sort

pandas/core/groupby/groupby.py

@@ -100,7 +100,10 @@ class providing the base-class of operations.
     numba_,
     ops,
 )
-from pandas.core.groupby.indexing import GroupByIndexingMixin
+from pandas.core.groupby.indexing import (
+    GroupByIndexingMixin,
+    GroupByNthSelector,
+)
 from pandas.core.indexes.api import (
     CategoricalIndex,
     Index,
@@ -902,6 +905,15 @@ def __getattr__(self, attr: str):
             f"'{type(self).__name__}' object has no attribute '{attr}'"
         )
 
+    def __getattribute__(self, attr: str):
+        # Intercept nth to allow both call and index
+        if attr == "nth":
+            return GroupByNthSelector(self)
+        elif attr == "nth_actual":
+            return super().__getattribute__("nth")
+        else:
+            return super().__getattribute__(attr)
+
     @final
     def _make_wrapper(self, name: str) -> Callable:
         assert name in self._apply_allowlist
@@ -2524,6 +2536,9 @@ def nth(
         """
         Take the nth row from each group if n is an int, otherwise a subset of rows.
 
+        Can be either a call or an index. dropna is not available with index notation.
+        Index notation accepts a comma separated list of integers and slices.
+
         If dropna, will take the nth non-null row, dropna is either
         'all' or 'any'; this is equivalent to calling dropna(how=dropna)
         before the groupby.
@@ -2535,6 +2550,7 @@ def nth(
 
             .. versionchanged:: 1.4.0
                 Added slice and lists containiing slices.
+                Added index notation.
 
         dropna : {'any', 'all', None}, default None
             Apply the specified dropna operation before counting which row is
@@ -2580,6 +2596,22 @@ def nth(
         1  2.0
         2  3.0
 
+        Index notation may also be used
+
+        >>> g.nth[0, 1]
+             B
+        A
+        1  NaN
+        1  2.0
+        2  3.0
+        2  5.0
+        >>> g.nth[:-1]
+             B
+        A
+        1  NaN
+        1  2.0
+        2  3.0
+
         Specifying `dropna` allows count ignoring ``NaN``
 
         >>> g.nth(0, dropna='any')

pandas/core/groupby/indexing.py

@@ -3,6 +3,7 @@
 from typing import (
     TYPE_CHECKING,
     Iterable,
+    Literal,
     cast,
 )
 
@@ -281,3 +282,22 @@ def __getitem__(self, arg: PositionalIndexer | tuple) -> DataFrame | Series:
         self.groupby_object._reset_group_selection()
         mask = self.groupby_object._make_mask_from_positional_indexer(arg)
         return self.groupby_object._mask_selected_obj(mask)
+
+
+class GroupByNthSelector:
+    """
+    Dynamically substituted for GroupBy.nth to enable both call and index
+    """
+
+    def __init__(self, groupby_object: groupby.GroupBy):
+        self.groupby_object = groupby_object
+
+    def __call__(
+        self,
+        n: PositionalIndexer | tuple,
+        dropna: Literal["any", "all", None] = None,
+    ) -> DataFrame | Series:
+        return self.groupby_object.nth_actual(n, dropna)
+
+    def __getitem__(self, n: PositionalIndexer | tuple) -> DataFrame | Series:
+        return self.groupby_object.nth_actual(n)

pandas/core/indexes/base.py

@@ -7016,7 +7016,7 @@ def _maybe_cast_data_without_dtype(
             "In a future version, the Index constructor will not infer numeric "
             "dtypes when passed object-dtype sequences (matching Series behavior)",
             FutureWarning,
-            stacklevel=find_stack_level(),
+            stacklevel=3,
         )
     if result.dtype.kind in ["b", "c"]:
         return subarr

pandas/io/excel/_base.py

@@ -1000,7 +1000,7 @@ def __init__(
 
         # cast ExcelWriter to avoid adding 'if self.handles is not None'
         self.handles = IOHandles(
-            cast(IO[bytes], path), compression={"copression": None}
+            cast(IO[bytes], path), compression={"compression": None}
         )
         if not isinstance(path, ExcelWriter):
             self.handles = get_handle(

pandas/tests/arrays/boolean/test_function.py

@@ -66,6 +66,17 @@ def test_ufuncs_unary(ufunc):
     tm.assert_series_equal(result, expected)
 
 
+def test_ufunc_numeric():
+    # np.sqrt on np.bool returns float16, which we upcast to Float32
+    #  bc we do not have Float16
+    arr = pd.array([True, False, None], dtype="boolean")
+
+    res = np.sqrt(arr)
+
+    expected = pd.array([1, 0, None], dtype="Float32")
+    tm.assert_extension_array_equal(res, expected)
+
+
 @pytest.mark.parametrize("values", [[True, False], [True, None]])
 def test_ufunc_reduce_raises(values):
     a = pd.array(values, dtype="boolean")

pandas/tests/arrays/categorical/test_operators.py

@@ -371,17 +371,15 @@ def test_numeric_like_ops(self):
         # min/max)
         s = df["value_group"]
         for op in ["kurt", "skew", "var", "std", "mean", "sum", "median"]:
-            msg = f"'Categorical' does not implement reduction '{op}'"
+            msg = f"does not support reduction '{op}'"
             with pytest.raises(TypeError, match=msg):
                 getattr(s, op)(numeric_only=False)
 
         # mad technically works because it takes always the numeric data
 
         # numpy ops
         s = Series(Categorical([1, 2, 3, 4]))
-        with pytest.raises(
-            TypeError, match="'Categorical' does not implement reduction 'sum'"
-        ):
+        with pytest.raises(TypeError, match="does not support reduction 'sum'"):
             np.sum(s)
 
         # numeric ops on a Series

pandas/tests/arrays/floating/test_construction.py

@@ -1,6 +1,13 @@
+import locale
+
 import numpy as np
 import pytest
 
+from pandas.compat import (
+    is_platform_windows,
+    np_version_under1p19,
+)
+
 import pandas as pd
 import pandas._testing as tm
 from pandas.core.arrays import FloatingArray
@@ -40,6 +47,33 @@ def test_floating_array_constructor():
         FloatingArray(values)
 
 
+def test_floating_array_disallows_float16(request):
+    # GH#44715
+    arr = np.array([1, 2], dtype=np.float16)
+    mask = np.array([False, False])
+
+    msg = "FloatingArray does not support np.float16 dtype"
+    with pytest.raises(TypeError, match=msg):
+        FloatingArray(arr, mask)
+
+    if not np_version_under1p19:
+        # Troubleshoot
+        #  https://github.com/numpy/numpy/issues/20512#issuecomment-985807740
+        lowered = np.core._type_aliases.english_lower("Float16")
+        assert lowered == "float16", lowered
+
+    if np_version_under1p19 or (
+        locale.getlocale()[0] != "en_US" and not is_platform_windows()
+    ):
+        # the locale condition may need to be refined; this fails on
+        #  the CI in the ZH_CN build
+        mark = pytest.mark.xfail(reason="numpy does not raise on np.dtype('Float16')")
+        request.node.add_marker(mark)
+
+    with pytest.raises(TypeError, match="data type 'Float16' not understood"):
+        pd.array([1.0, 2.0], dtype="Float16")
+
+
 def test_floating_array_constructor_copy():
     values = np.array([1, 2, 3, 4], dtype="float64")
     mask = np.array([False, False, False, True], dtype="bool")