Skip to content

Catch Exception in combine #22936

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 15 commits into from
Oct 4, 2018
4 changes: 2 additions & 2 deletions pandas/core/arrays/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -777,8 +777,8 @@ def convert_values(param):
if coerce_to_dtype:
try:
res = self._from_sequence(res)
except TypeError:
pass
except Exception:
res = np.asarray(res, dtype=object)
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm, this looks like it's in the wrong place.

Right now with coerce_to_dtype=False we'd return an list, not an ndarray. I assume we want an ndarray of objects regardless of coerce_to_dtype? I don't want the return type to be Union[ExtensionArray, List, ndarray]

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Grr... ok what do to here?

If we do

                res = np.asarray(res, dtype=object)

we'll start failing some other tests like Series[DecimalArray] == 1, since we get an object dtype instead of bool. So we just leave it up to NumPy's regular inference then, res = np.asarray(res)?


return res

Expand Down
4 changes: 3 additions & 1 deletion pandas/core/series.py
Original file line number Diff line number Diff line change
Expand Up @@ -2324,9 +2324,11 @@ def combine(self, other, func, fill_value=None):
elif is_extension_array_dtype(self.values):
# The function can return something of any type, so check
# if the type is compatible with the calling EA
# ExtensionArray._from_sequence can raise anything, so we
# have to catch everything.
try:
new_values = self._values._from_sequence(new_values)
except TypeError:
except Exception:
pass

return self._constructor(new_values, index=new_index, name=new_name)
Expand Down
4 changes: 4 additions & 0 deletions pandas/tests/extension/decimal/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
from .array import DecimalArray, DecimalDtype, to_decimal, make_data


__all__ = ['DecimalArray', 'DecimalDtype', 'to_decimal', 'make_data']
9 changes: 9 additions & 0 deletions pandas/tests/extension/decimal/array.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import decimal
import numbers
import random
import sys

import numpy as np
Expand Down Expand Up @@ -138,5 +139,13 @@ def _concat_same_type(cls, to_concat):
return cls(np.concatenate([x._data for x in to_concat]))


def to_decimal(values, context=None):
return DecimalArray([decimal.Decimal(x) for x in values], context=context)


def make_data():
return [decimal.Decimal(random.random()) for _ in range(100)]


DecimalArray._add_arithmetic_ops()
DecimalArray._add_comparison_ops()
39 changes: 33 additions & 6 deletions pandas/tests/extension/decimal/test_decimal.py
Original file line number Diff line number Diff line change
@@ -1,18 +1,14 @@
import operator
import decimal

import random
import numpy as np
import pandas as pd
import pandas.util.testing as tm
import pytest

from pandas.tests.extension import base

from .array import DecimalDtype, DecimalArray


def make_data():
return [decimal.Decimal(random.random()) for _ in range(100)]
from .array import DecimalDtype, DecimalArray, make_data


@pytest.fixture
Expand Down Expand Up @@ -275,3 +271,34 @@ def test_compare_array(self, data, all_compare_operators):
other = pd.Series(data) * [decimal.Decimal(pow(2.0, i))
for i in alter]
self._compare_other(s, data, op_name, other)


class DecimalArrayWithoutFromSequence(DecimalArray):
"""Helper class for testing error handling in _from_sequence."""
def _from_sequence(cls, scalars, dtype=None, copy=False):
raise KeyError("For the test")


def test_combine_from_sequence_raises():
# https://github.com/pandas-dev/pandas/issues/22850
ser = pd.Series(DecimalArrayWithoutFromSequence([
decimal.Decimal("1.0"),
decimal.Decimal("2.0")
]))
result = ser.combine(ser, operator.add)

# note: object dtype
expected = pd.Series([decimal.Decimal("2.0"),
decimal.Decimal("4.0")], dtype="object")
tm.assert_series_equal(result, expected)


def test_scalar_ops_from_sequence_raises():
arr = DecimalArrayWithoutFromSequence([
decimal.Decimal("1.0"),
decimal.Decimal("2.0")
])
result = arr + arr
expected = np.array([decimal.Decimal("2.0"), decimal.Decimal("4.0")],
dtype="object")
tm.assert_numpy_array_equal(result, expected)
3 changes: 3 additions & 0 deletions pandas/tests/extension/json/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
from .array import JSONArray, JSONDtype, make_data

__all__ = ['JSONArray', 'JSONDtype', 'make_data']
9 changes: 9 additions & 0 deletions pandas/tests/extension/json/array.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@
import collections
import itertools
import numbers
import random
import string
import sys

import numpy as np
Expand Down Expand Up @@ -179,3 +181,10 @@ def _values_for_argsort(self):
# cast them to an (N, P) array, instead of an (N,) array of tuples.
frozen = [()] + [tuple(x.items()) for x in self]
return np.array(frozen, dtype=object)[1:]


def make_data():
# TODO: Use a regular dict. See _NDFrameIndexer._setitem_with_indexer
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is pretty odd here

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@jorisvandenbossche requested these be exported, since they're useful for creating datasets. It'd be more strange to import it from the test file.

return [collections.UserDict([
(random.choice(string.ascii_letters), random.randint(0, 100))
for _ in range(random.randint(0, 10))]) for _ in range(100)]
11 changes: 1 addition & 10 deletions pandas/tests/extension/json/test_json.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
import operator
import collections
import random
import string

import pytest

Expand All @@ -10,18 +8,11 @@
from pandas.compat import PY2, PY36
from pandas.tests.extension import base

from .array import JSONArray, JSONDtype
from .array import JSONArray, JSONDtype, make_data

pytestmark = pytest.mark.skipif(PY2, reason="Py2 doesn't have a UserDict")


def make_data():
# TODO: Use a regular dict. See _NDFrameIndexer._setitem_with_indexer
return [collections.UserDict([
(random.choice(string.ascii_letters), random.randint(0, 100))
for _ in range(random.randint(0, 10))]) for _ in range(100)]


@pytest.fixture
def dtype():
return JSONDtype()
Expand Down