From 11b6c6b4a98b241b267aefb3ffce59327879079f Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Thu, 14 Nov 2019 19:07:44 -0800 Subject: [PATCH 1/3] TST: fix DecimalArray._reduce kludges --- pandas/core/groupby/generic.py | 7 ------- pandas/core/groupby/groupby.py | 3 --- pandas/tests/extension/decimal/array.py | 15 +++++++++++++-- pandas/tests/extension/decimal/test_decimal.py | 2 +- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/pandas/core/groupby/generic.py b/pandas/core/groupby/generic.py index 002d8640f109d..88f7d3d64d366 100644 --- a/pandas/core/groupby/generic.py +++ b/pandas/core/groupby/generic.py @@ -897,13 +897,6 @@ def aggregate(self, func=None, *args, **kwargs): # raised directly by _aggregate_multiple_funcs raise result = self._aggregate_frame(func) - except NotImplementedError as err: - if "decimal does not support skipna=True" in str(err): - # FIXME: kludge for DecimalArray tests - pass - else: - raise - result = self._aggregate_frame(func) else: result.columns = Index( result.columns.levels[0], name=self._selected_obj.columns.name diff --git a/pandas/core/groupby/groupby.py b/pandas/core/groupby/groupby.py index 280f1e88b0ea8..d23c16c989c48 100644 --- a/pandas/core/groupby/groupby.py +++ b/pandas/core/groupby/groupby.py @@ -1349,9 +1349,6 @@ def f(self, **kwargs): # raised in _get_cython_function, in some cases can # be trimmed by implementing cython funcs for more dtypes pass - elif "decimal does not support skipna=True" in str(err): - # FIXME: kludge for test_decimal:test_in_numeric_groupby - pass else: raise diff --git a/pandas/tests/extension/decimal/array.py b/pandas/tests/extension/decimal/array.py index 93816e3a8a613..f03994bc199a4 100644 --- a/pandas/tests/extension/decimal/array.py +++ b/pandas/tests/extension/decimal/array.py @@ -71,7 +71,15 @@ def dtype(self): @classmethod def _from_sequence(cls, scalars, dtype=None, copy=False): - return cls(scalars) + try: + return cls(scalars) + except TypeError: + # handling int 0 is necessary for groupby tests + if isinstance(scalars, np.ndarray): + for i, val in enumerate(scalars): + if isinstance(val, (int, float)): + scalars[i] = decimal.Decimal(val) + return cls(scalars) @classmethod def _from_sequence_of_strings(cls, strings, dtype=None, copy=False): @@ -166,7 +174,10 @@ def _concat_same_type(cls, to_concat): def _reduce(self, name, skipna=True, **kwargs): if skipna: - raise NotImplementedError("decimal does not support skipna=True") + # If we don't have any NAs, we can ignore skipna + if self.isna().any(): + other = self[~self.isna()] + return other._reduce(name, **kwargs) try: op = getattr(self.data, name) diff --git a/pandas/tests/extension/decimal/test_decimal.py b/pandas/tests/extension/decimal/test_decimal.py index 86724d4d09819..ce819c13c4498 100644 --- a/pandas/tests/extension/decimal/test_decimal.py +++ b/pandas/tests/extension/decimal/test_decimal.py @@ -145,7 +145,7 @@ class TestMissing(BaseDecimal, base.BaseMissingTests): class Reduce: def check_reduce(self, s, op_name, skipna): - if skipna or op_name in ["median", "skew", "kurt"]: + if op_name in ["median", "skew", "kurt"]: with pytest.raises(NotImplementedError): getattr(s, op_name)(skipna=skipna) From 331c5a963940080a3c58bdf6305eb61af4f6b7da Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Thu, 14 Nov 2019 21:02:31 -0800 Subject: [PATCH 2/3] Fix incorrect behavior in old numpy --- pandas/core/groupby/ops.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pandas/core/groupby/ops.py b/pandas/core/groupby/ops.py index 7ed79e4b00371..35aa62e712afd 100644 --- a/pandas/core/groupby/ops.py +++ b/pandas/core/groupby/ops.py @@ -663,6 +663,14 @@ def _aggregate_series_pure_python(self, obj: Series, func): counts[label] = group.shape[0] result[label] = res + if isinstance(res, np.bool_): + if is_extension_array_dtype(group.dtype): + # FIXME: kludge for DecimalArray on numpy<1.15 returning + # np.bool_(False) when we have all-NA DecimalArray + if group.isna().all(): + res = group.dtype.na_value + result[label] = res + assert result is not None result = lib.maybe_convert_objects(result, try_float=0) # TODO: try_cast back to EA? From d93ed3718946ab69e9262d1d40ad823ce6322af2 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Fri, 15 Nov 2019 09:05:25 -0800 Subject: [PATCH 3/3] fix in _reduce --- pandas/core/groupby/ops.py | 8 -------- pandas/tests/extension/decimal/array.py | 14 +++++--------- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/pandas/core/groupby/ops.py b/pandas/core/groupby/ops.py index 35aa62e712afd..7ed79e4b00371 100644 --- a/pandas/core/groupby/ops.py +++ b/pandas/core/groupby/ops.py @@ -663,14 +663,6 @@ def _aggregate_series_pure_python(self, obj: Series, func): counts[label] = group.shape[0] result[label] = res - if isinstance(res, np.bool_): - if is_extension_array_dtype(group.dtype): - # FIXME: kludge for DecimalArray on numpy<1.15 returning - # np.bool_(False) when we have all-NA DecimalArray - if group.isna().all(): - res = group.dtype.na_value - result[label] = res - assert result is not None result = lib.maybe_convert_objects(result, try_float=0) # TODO: try_cast back to EA? diff --git a/pandas/tests/extension/decimal/array.py b/pandas/tests/extension/decimal/array.py index f03994bc199a4..f9ba4b7a8ba16 100644 --- a/pandas/tests/extension/decimal/array.py +++ b/pandas/tests/extension/decimal/array.py @@ -71,15 +71,7 @@ def dtype(self): @classmethod def _from_sequence(cls, scalars, dtype=None, copy=False): - try: - return cls(scalars) - except TypeError: - # handling int 0 is necessary for groupby tests - if isinstance(scalars, np.ndarray): - for i, val in enumerate(scalars): - if isinstance(val, (int, float)): - scalars[i] = decimal.Decimal(val) - return cls(scalars) + return cls(scalars) @classmethod def _from_sequence_of_strings(cls, strings, dtype=None, copy=False): @@ -179,6 +171,10 @@ def _reduce(self, name, skipna=True, **kwargs): other = self[~self.isna()] return other._reduce(name, **kwargs) + if name == "sum" and len(self) == 0: + # GH#29630 avoid returning int 0 or np.bool_(False) on old numpy + return decimal.Decimal(0) + try: op = getattr(self.data, name) except AttributeError: