From 619d2bbc4ffb689085097659a8895c81d51cd80e Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 20 Feb 2018 09:02:05 -0800 Subject: [PATCH 1/7] split bool ops test, parametrize where possible --- pandas/tests/series/test_operators.py | 213 ++++++++++++++++---------- 1 file changed, 128 insertions(+), 85 deletions(-) diff --git a/pandas/tests/series/test_operators.py b/pandas/tests/series/test_operators.py index 554b3e15d8f10..e6500fd9a4a5b 100644 --- a/pandas/tests/series/test_operators.py +++ b/pandas/tests/series/test_operators.py @@ -183,46 +183,47 @@ def test_comparison_tuples(self): expected = Series([True, False]) assert_series_equal(result, expected) - def test_comparison_operators_with_nas(self): + @pytest.mark.parametrize('op', ['lt', 'le', 'gt', 'ge', 'eq', 'ne']) + def test_comparison_operators_with_nas(self, op): ser = Series(bdate_range('1/1/2000', periods=10), dtype=object) ser[::2] = np.nan # test that comparisons work - ops = ['lt', 'le', 'gt', 'ge', 'eq', 'ne'] - for op in ops: - val = ser[5] + val = ser[5] - f = getattr(operator, op) - result = f(ser, val) + f = getattr(operator, op) + result = f(ser, val) - expected = f(ser.dropna(), val).reindex(ser.index) + expected = f(ser.dropna(), val).reindex(ser.index) - if op == 'ne': - expected = expected.fillna(True).astype(bool) - else: - expected = expected.fillna(False).astype(bool) + if op == 'ne': + expected = expected.fillna(True).astype(bool) + else: + expected = expected.fillna(False).astype(bool) - assert_series_equal(result, expected) + assert_series_equal(result, expected) - # fffffffuuuuuuuuuuuu - # result = f(val, s) - # expected = f(val, s.dropna()).reindex(s.index) - # assert_series_equal(result, expected) + # fffffffuuuuuuuuuuuu + # result = f(val, s) + # expected = f(val, s.dropna()).reindex(s.index) + # assert_series_equal(result, expected) - # boolean &, |, ^ should work with object arrays and propagate NAs + @pytest.mark.parametrize('bool_op', ['and_', 'or_', 'xor']) + def test_bool_operators_with_nas(self, bool_op): + # boolean &, |, ^ should work with object arrays and propagate NAs - ops = ['and_', 'or_', 'xor'] - mask = ser.isna() - for bool_op in ops: - func = getattr(operator, bool_op) + ser = Series(bdate_range('1/1/2000', periods=10), dtype=object) + ser[::2] = np.nan - filled = ser.fillna(ser[0]) + func = getattr(operator, bool_op) + result = func(ser < ser[9], ser > ser[3]) - result = func(ser < ser[9], ser > ser[3]) + mask = ser.isna() + filled = ser.fillna(ser[0]) + expected = func(filled < filled[9], filled > filled[3]) + expected[mask] = False - expected = func(filled < filled[9], filled > filled[3]) - expected[mask] = False - assert_series_equal(result, expected) + assert_series_equal(result, expected) def test_comparison_object_numeric_nas(self): ser = Series(np.random.randn(10), dtype=object) @@ -376,10 +377,10 @@ def test_comparison_different_length(self): b = Series([2, 3, 4]) pytest.raises(ValueError, a.__eq__, b) - def test_comparison_label_based(self): + def test_bool_opslabel_based(self): # GH 4947 - # comparisons should be label based + # boolean ops should be label based a = Series([True, False, True], list('bca')) b = Series([False, True, False], list('abc')) @@ -1823,128 +1824,169 @@ def test_ops_datetimelike_align(self): def test_operators_bitwise(self): # GH 9016: support bitwise op for integer types - index = list('bca') + s_0123 = Series(range(4), dtype='int64') + s_1111 = Series([1] * 4, dtype='int8') - s_tft = Series([True, False, True], index=index) - s_fff = Series([False, False, False], index=index) - s_tff = Series([True, False, False], index=index) - s_empty = Series([]) + # We cannot wrap s111.astype(np.int32) with pd.Index because it + # will case to int64 + res = s_0123.astype(np.int16) | s_1111.astype(np.int32) + expected = Series([1, 1, 3, 3], dtype='int32') + assert_series_equal(res, expected) - # TODO: unused - # s_0101 = Series([0, 1, 0, 1]) + def test_operators_bitwise_invalid(self): + # GH#9016: support bitwise op for integer types + s_0123 = Series(range(4), dtype='int64') + s_1111 = Series([1] * 4, dtype='int8') + + with pytest.raises(TypeError): + s_1111 & 'a' + with pytest.raises(TypeError): + s_1111 & ['a', 'b', 'c', 'd'] + with pytest.raises(TypeError): + s_0123 & 3.14 + with pytest.raises(TypeError): + s_0123 & [0.1, 4, 3.14, 2] + def test_operators_bitwise_int_series_with_float_series(self): + # GH#9016: support bitwise op for integer types + s_0123 = Series(range(4), dtype='int64') + s_ftft = Series([False, True, False, True]) + result = s_0123 & Series([0.1, 4, -3.14, 2]) + assert_series_equal(result, s_ftft) + + @pytest.mark.parametrize('box', [np.array, pd.Index, pd.Series]) + def test_operators_bitwise_with_int_arraylike(self, box): + # GH#9016: support bitwise op for integer types + # GH#??? allow for operating with Index s_0123 = Series(range(4), dtype='int64') s_3333 = Series([3] * 4) s_4444 = Series([4] * 4) - res = s_tft & s_empty - expected = s_fff + res = s_0123 & box(s_3333) + expected = Series(range(4), dtype='int64') assert_series_equal(res, expected) - res = s_tft | s_empty - expected = s_tft + res = s_0123 | box(s_4444) + expected = Series(range(4, 8), dtype='int64') assert_series_equal(res, expected) - res = s_0123 & s_3333 - expected = Series(range(4), dtype='int64') + s_1111 = Series([1] * 4, dtype='int8') + res = s_0123 & box(s_1111) + expected = Series([0, 1, 0, 1], dtype='int64') assert_series_equal(res, expected) - res = s_0123 | s_4444 - expected = Series(range(4, 8), dtype='int64') - assert_series_equal(res, expected) + def test_operators_bitwise_with_bool(self): + # GH#9016: support bitwise op for integer types + s_0123 = Series(range(4), dtype='int64') - s_a0b1c0 = Series([1], list('b')) + assert_series_equal(s_0123 & False, Series([False] * 4)) + assert_series_equal(s_0123 ^ False, Series([False, True, True, True])) + assert_series_equal(s_0123 & [False], Series([False] * 4)) + assert_series_equal(s_0123 & (False), Series([False] * 4)) - res = s_tft & s_a0b1c0 - expected = s_tff.reindex(list('abc')) - assert_series_equal(res, expected) + def test_operators_bitwise_with_integers(self): + # GH#9016: support bitwise op for integer types + index = list('bca') - res = s_tft | s_a0b1c0 - expected = s_tft.reindex(list('abc')) - assert_series_equal(res, expected) + s_tft = Series([True, False, True], index=index) + s_fff = Series([False, False, False], index=index) + s_0123 = Series(range(4), dtype='int64') - n0 = 0 - res = s_tft & n0 + res = s_tft & 0 expected = s_fff assert_series_equal(res, expected) - res = s_0123 & n0 + res = s_0123 & 0 expected = Series([0] * 4) assert_series_equal(res, expected) - n1 = 1 - res = s_tft & n1 + res = s_tft & 1 expected = s_tft assert_series_equal(res, expected) - res = s_0123 & n1 + res = s_0123 & 1 expected = Series([0, 1, 0, 1]) assert_series_equal(res, expected) - s_1111 = Series([1] * 4, dtype='int8') - res = s_0123 & s_1111 - expected = Series([0, 1, 0, 1], dtype='int64') + def test_operators_bitwise_with_reindexing(self): + # ops where reindeing needs to be done, so operating with a Series + # makes sense but with an Index or array does not + # GH#9016: support bitwise op for integer types + index = list('bca') + + s_tft = Series([True, False, True], index=index) + s_fff = Series([False, False, False], index=index) + s_tff = Series([True, False, False], index=index) + s_empty = Series([]) + s_a0b1c0 = Series([1], list('b')) + s_0123 = Series(range(4), dtype='int64') + + res = s_tft & s_empty + expected = s_fff assert_series_equal(res, expected) - res = s_0123.astype(np.int16) | s_1111.astype(np.int32) - expected = Series([1, 1, 3, 3], dtype='int32') + res = s_tft | s_empty + expected = s_tft + assert_series_equal(res, expected) + + res = s_tft & s_a0b1c0 + expected = s_tff.reindex(list('abc')) assert_series_equal(res, expected) - pytest.raises(TypeError, lambda: s_1111 & 'a') - pytest.raises(TypeError, lambda: s_1111 & ['a', 'b', 'c', 'd']) - pytest.raises(TypeError, lambda: s_0123 & np.NaN) - pytest.raises(TypeError, lambda: s_0123 & 3.14) - pytest.raises(TypeError, lambda: s_0123 & [0.1, 4, 3.14, 2]) + res = s_tft | s_a0b1c0 + expected = s_tft.reindex(list('abc')) + assert_series_equal(res, expected) # s_0123 will be all false now because of reindexing like s_tft if compat.PY3: # unable to sort incompatible object via .union. exp = Series([False] * 7, index=['b', 'c', 'a', 0, 1, 2, 3]) with tm.assert_produces_warning(RuntimeWarning): - assert_series_equal(s_tft & s_0123, exp) + result = s_tft & s_0123 + assert_series_equal(result, exp) else: exp = Series([False] * 7, index=[0, 1, 2, 3, 'a', 'b', 'c']) - assert_series_equal(s_tft & s_0123, exp) + result = s_tft & s_0123 + assert_series_equal(result, exp) # s_tft will be all false now because of reindexing like s_0123 if compat.PY3: # unable to sort incompatible object via .union. exp = Series([False] * 7, index=[0, 1, 2, 3, 'b', 'c', 'a']) with tm.assert_produces_warning(RuntimeWarning): + result = s_0123 & s_tft assert_series_equal(s_0123 & s_tft, exp) else: exp = Series([False] * 7, index=[0, 1, 2, 3, 'a', 'b', 'c']) assert_series_equal(s_0123 & s_tft, exp) - assert_series_equal(s_0123 & False, Series([False] * 4)) - assert_series_equal(s_0123 ^ False, Series([False, True, True, True])) - assert_series_equal(s_0123 & [False], Series([False] * 4)) - assert_series_equal(s_0123 & (False), Series([False] * 4)) - assert_series_equal(s_0123 & Series([False, np.NaN, False, False]), - Series([False] * 4)) + def test_operators_bitwise_with_nans(self): + # with NaNs present op(series, series) works but op(series, index) + # is not implemented + # GH#9016: support bitwise op for integer types + s_0123 = Series(range(4), dtype='int64') + result = s_0123 & Series([False, np.NaN, False, False]) + assert_series_equal(result, Series([False] * 4)) s_ftft = Series([False, True, False, True]) - assert_series_equal(s_0123 & Series([0.1, 4, -3.14, 2]), s_ftft) - s_abNd = Series(['a', 'b', np.NaN, 'd']) - res = s_0123 & s_abNd + result = s_0123 & s_abNd expected = s_ftft - assert_series_equal(res, expected) + assert_series_equal(result, expected) def test_scalar_na_cmp_corners(self): s = Series([2, 3, 4, 5, 6, 7, 8, 9, 10]) - def tester(a, b): - return a & b - - pytest.raises(TypeError, tester, s, datetime(2005, 1, 1)) + with pytest.raises(TypeError): + s & datetime(2005, 1, 1) s = Series([2, 3, 4, 5, 6, 7, 8, 9, datetime(2005, 1, 1)]) s[::2] = np.nan expected = Series(True, index=s.index) expected[::2] = False - assert_series_equal(tester(s, list(s)), expected) + result = s & list(s) + assert_series_equal(result, expected) d = DataFrame({'A': s}) # TODO: Fix this exception - needs to be fixed! (see GH5035) @@ -1955,7 +1997,8 @@ def tester(a, b): # https://github.com/pandas-dev/pandas/issues/5284 pytest.raises(ValueError, lambda: d.__and__(s, axis='columns')) - pytest.raises(ValueError, tester, s, d) + with pytest.raises(ValueError): + s & d # this is wrong as its not a boolean result # result = d.__and__(s,axis='index') From 7cf9a7a4ef0f3e5393100a5f4be6708b8f126292 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 20 Feb 2018 09:03:40 -0800 Subject: [PATCH 2/7] remove non-standard imports of nan, inf --- pandas/tests/series/test_operators.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/pandas/tests/series/test_operators.py b/pandas/tests/series/test_operators.py index e6500fd9a4a5b..0b7a489824fa0 100644 --- a/pandas/tests/series/test_operators.py +++ b/pandas/tests/series/test_operators.py @@ -9,7 +9,6 @@ import operator from itertools import product, starmap -from numpy import nan, inf import numpy as np import pandas as pd @@ -670,7 +669,7 @@ def test_floordiv_div(self): ser = Series([-1, 0, 1], name='first') result = ser // 0 - expected = Series([-inf, nan, inf], name='first') + expected = Series([-np.inf, np.nan, np.inf], name='first') assert_series_equal(result, expected) @@ -1069,9 +1068,9 @@ def test_timedelta64_ops_nat(self): assert_series_equal(1.5 * timedelta_series, Series([NaT, Timedelta('1.5s')])) - assert_series_equal(timedelta_series * nan, + assert_series_equal(timedelta_series * np.nan, nat_series_dtype_timedelta) - assert_series_equal(nan * timedelta_series, + assert_series_equal(np.nan * timedelta_series, nat_series_dtype_timedelta) # division @@ -1079,7 +1078,7 @@ def test_timedelta64_ops_nat(self): Series([NaT, Timedelta('0.5s')])) assert_series_equal(timedelta_series / 2.0, Series([NaT, Timedelta('0.5s')])) - assert_series_equal(timedelta_series / nan, + assert_series_equal(timedelta_series / np.nan, nat_series_dtype_timedelta) def test_td64_sub_NaT(self): @@ -2183,12 +2182,12 @@ def _check_fill(meth, op, a, b, fill_value=0): with np.errstate(all='ignore'): if amask[i]: if bmask[i]: - exp_values.append(nan) + exp_values.append(np.nan) continue exp_values.append(op(fill_value, b[i])) elif bmask[i]: if amask[i]: - exp_values.append(nan) + exp_values.append(np.nan) continue exp_values.append(op(a[i], fill_value)) else: @@ -2198,8 +2197,8 @@ def _check_fill(meth, op, a, b, fill_value=0): expected = Series(exp_values, exp_index) assert_series_equal(result, expected) - a = Series([nan, 1., 2., 3., nan], index=np.arange(5)) - b = Series([nan, 1, nan, 3, nan, 4.], index=np.arange(6)) + a = Series([np.nan, 1., 2., 3., np.nan], index=np.arange(5)) + b = Series([np.nan, 1, np.nan, 3, np.nan, 4.], index=np.arange(6)) pairings = [] for op in ['add', 'sub', 'mul', 'pow', 'truediv', 'floordiv']: From f5b262b41cc2dc5c494abc250cb8106d982bed23 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 20 Feb 2018 10:55:31 -0800 Subject: [PATCH 3/7] simplify bool ops closures --- doc/source/whatsnew/v0.23.0.txt | 1 + pandas/core/ops.py | 71 +++++++++++++++------------ pandas/tests/series/test_operators.py | 11 +++++ 3 files changed, 51 insertions(+), 32 deletions(-) diff --git a/doc/source/whatsnew/v0.23.0.txt b/doc/source/whatsnew/v0.23.0.txt index 7bd47c7172671..a728aee016ab6 100644 --- a/doc/source/whatsnew/v0.23.0.txt +++ b/doc/source/whatsnew/v0.23.0.txt @@ -866,3 +866,4 @@ Other ^^^^^ - Improved error message when attempting to use a Python keyword as an identifier in a ``numexpr`` backed query (:issue:`18221`) +- Boolean operations ``&, |, ^`` between a ``Series`` and an ``Index`` will no longer raise ``TypeError`` (:issue:`19792`) diff --git a/pandas/core/ops.py b/pandas/core/ops.py index da65f1f31ed2a..02e51b7a92ac0 100644 --- a/pandas/core/ops.py +++ b/pandas/core/ops.py @@ -1058,69 +1058,76 @@ def _bool_method_SERIES(cls, op, special): Wrapper function for Series arithmetic operations, to avoid code duplication. """ + name = _get_op_name(op, special) def na_op(x, y): try: result = op(x, y) except TypeError: - if isinstance(y, list): - y = construct_1d_object_array_from_listlike(y) - - if isinstance(y, (np.ndarray, ABCSeries)): - if (is_bool_dtype(x.dtype) and is_bool_dtype(y.dtype)): - result = op(x, y) # when would this be hit? - else: - x = _ensure_object(x) - y = _ensure_object(y) - result = lib.vec_binop(x, y, op) + assert not isinstance(y, (list, ABCSeries, pd.Index)) + if isinstance(y, np.ndarray): + # This next assertion is here because there used to be + # a branch that specifically caught this case. + assert not (is_bool_dtype(x) and is_bool_dtype(y)) + x = _ensure_object(x) + y = _ensure_object(y) + result = lib.vec_binop(x, y, op) else: # let null fall thru + assert is_scalar(y) if not isna(y): y = bool(y) try: result = lib.scalar_binop(x, y, op) except: - msg = ("cannot compare a dtyped [{dtype}] array " - "with a scalar of type [{type}]" - ).format(dtype=x.dtype, type=type(y).__name__) - raise TypeError(msg) + raise TypeError("cannot compare a dtyped [{dtype}] array " + "with a scalar of type [{typ}]" + .format(dtype=x.dtype, + typ=type(y).__name__)) return result + fill_int = lambda x: x.fillna(0) + fill_bool = lambda x: x.fillna(False).astype(bool) + def wrapper(self, other): is_self_int_dtype = is_integer_dtype(self.dtype) - fill_int = lambda x: x.fillna(0) - fill_bool = lambda x: x.fillna(False).astype(bool) - self, other = _align_method_SERIES(self, other, align_asobject=True) + res_name = _get_series_op_result_name(self, other) + if isinstance(other, ABCDataFrame): # Defer to DataFrame implementation; fail early return NotImplemented - elif isinstance(other, ABCSeries): - name = com._maybe_match_name(self, other) + elif isinstance(other, (ABCSeries, pd.Index)): is_other_int_dtype = is_integer_dtype(other.dtype) other = fill_int(other) if is_other_int_dtype else fill_bool(other) - - filler = (fill_int if is_self_int_dtype and is_other_int_dtype - else fill_bool) - - res_values = na_op(self.values, other.values) - unfilled = self._constructor(res_values, - index=self.index, name=name) - return filler(unfilled) + ovalues = other.values else: # scalars, list, tuple, np.array - filler = (fill_int if is_self_int_dtype and - is_integer_dtype(np.asarray(other)) else fill_bool) + is_other_int_dtype = is_integer_dtype(np.asarray(other)) + if isinstance(other, list): + # TODO: Can we do this before the is_integer_dtype check? + # could the is_integer_dtype check be checking the wrong + # thing? e.g. other = [[0, 1], [2, 3], [4, 5]]? + other = construct_1d_object_array_from_listlike(other) + ovalues = other - res_values = na_op(self.values, other) - unfilled = self._constructor(res_values, index=self.index) - return filler(unfilled).__finalize__(self) + filler = (fill_int if is_self_int_dtype and is_other_int_dtype + else fill_bool) + res_values = na_op(self.values, ovalues) + unfilled = self._constructor(res_values, index=self.index, + name=res_name) + result = filler(unfilled) + if not isinstance(other, ABCSeries): + result = result.__finalize__(self) + return result + + wrapper.__name__ = name return wrapper diff --git a/pandas/tests/series/test_operators.py b/pandas/tests/series/test_operators.py index 0b7a489824fa0..0fb0fa320ccf3 100644 --- a/pandas/tests/series/test_operators.py +++ b/pandas/tests/series/test_operators.py @@ -1853,6 +1853,17 @@ def test_operators_bitwise_int_series_with_float_series(self): result = s_0123 & Series([0.1, 4, -3.14, 2]) assert_series_equal(result, s_ftft) + @pytest.mark.xfail(reason='GH#19792 Series op doesnt support categorical') + def test_operators_bitwise_with_int_categorical(self): + # GH#9016: support bitwise op for integer types + # GH#??? allow for operating with Index + s_0123 = Series(range(4), dtype='int64').astype('category') + s_3333 = Series([3] * 4).astype('category') + + res = s_0123 & pd.Categorical(s_3333) + expected = Series(range(4), dtype='int64').astype('category') + assert_series_equal(res, expected) + @pytest.mark.parametrize('box', [np.array, pd.Index, pd.Series]) def test_operators_bitwise_with_int_arraylike(self, box): # GH#9016: support bitwise op for integer types From c03cc553c5980f17f45897bda6aad36536076735 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 20 Feb 2018 17:21:15 -0800 Subject: [PATCH 4/7] move whatsnew to breaking changes, revert non-central test diff --- doc/source/whatsnew/v0.23.0.txt | 2 +- pandas/tests/series/test_operators.py | 51 +++++++++++---------------- 2 files changed, 21 insertions(+), 32 deletions(-) diff --git a/doc/source/whatsnew/v0.23.0.txt b/doc/source/whatsnew/v0.23.0.txt index a728aee016ab6..d926d08db4eb3 100644 --- a/doc/source/whatsnew/v0.23.0.txt +++ b/doc/source/whatsnew/v0.23.0.txt @@ -585,6 +585,7 @@ Other API Changes - Set operations (union, difference...) on :class:`IntervalIndex` with incompatible index types will now raise a ``TypeError`` rather than a ``ValueError`` (:issue:`19329`) - :class:`DateOffset` objects render more simply, e.g. "" instead of "" (:issue:`19403`) - :func:`pandas.merge` provides a more informative error message when trying to merge on timezone-aware and timezone-naive columns (:issue:`15800`) +- Boolean operations ``&, |, ^`` between a ``Series`` and an ``Index`` will no longer raise ``TypeError`` (:issue:`19792`) .. _whatsnew_0230.deprecations: @@ -866,4 +867,3 @@ Other ^^^^^ - Improved error message when attempting to use a Python keyword as an identifier in a ``numexpr`` backed query (:issue:`18221`) -- Boolean operations ``&, |, ^`` between a ``Series`` and an ``Index`` will no longer raise ``TypeError`` (:issue:`19792`) diff --git a/pandas/tests/series/test_operators.py b/pandas/tests/series/test_operators.py index 0fb0fa320ccf3..2f4627b74a58b 100644 --- a/pandas/tests/series/test_operators.py +++ b/pandas/tests/series/test_operators.py @@ -9,6 +9,7 @@ import operator from itertools import product, starmap +from numpy import nan, inf import numpy as np import pandas as pd @@ -669,7 +670,7 @@ def test_floordiv_div(self): ser = Series([-1, 0, 1], name='first') result = ser // 0 - expected = Series([-np.inf, np.nan, np.inf], name='first') + expected = Series([-inf, nan, inf], name='first') assert_series_equal(result, expected) @@ -1068,9 +1069,9 @@ def test_timedelta64_ops_nat(self): assert_series_equal(1.5 * timedelta_series, Series([NaT, Timedelta('1.5s')])) - assert_series_equal(timedelta_series * np.nan, + assert_series_equal(timedelta_series * nan, nat_series_dtype_timedelta) - assert_series_equal(np.nan * timedelta_series, + assert_series_equal(nan * timedelta_series, nat_series_dtype_timedelta) # division @@ -1078,7 +1079,7 @@ def test_timedelta64_ops_nat(self): Series([NaT, Timedelta('0.5s')])) assert_series_equal(timedelta_series / 2.0, Series([NaT, Timedelta('0.5s')])) - assert_series_equal(timedelta_series / np.nan, + assert_series_equal(timedelta_series / nan, nat_series_dtype_timedelta) def test_td64_sub_NaT(self): @@ -1853,21 +1854,10 @@ def test_operators_bitwise_int_series_with_float_series(self): result = s_0123 & Series([0.1, 4, -3.14, 2]) assert_series_equal(result, s_ftft) - @pytest.mark.xfail(reason='GH#19792 Series op doesnt support categorical') - def test_operators_bitwise_with_int_categorical(self): - # GH#9016: support bitwise op for integer types - # GH#??? allow for operating with Index - s_0123 = Series(range(4), dtype='int64').astype('category') - s_3333 = Series([3] * 4).astype('category') - - res = s_0123 & pd.Categorical(s_3333) - expected = Series(range(4), dtype='int64').astype('category') - assert_series_equal(res, expected) - @pytest.mark.parametrize('box', [np.array, pd.Index, pd.Series]) def test_operators_bitwise_with_int_arraylike(self, box): # GH#9016: support bitwise op for integer types - # GH#??? allow for operating with Index + # GH#19795 allow for operating with Index s_0123 = Series(range(4), dtype='int64') s_3333 = Series([3] * 4) s_4444 = Series([4] * 4) @@ -1902,19 +1892,21 @@ def test_operators_bitwise_with_integers(self): s_fff = Series([False, False, False], index=index) s_0123 = Series(range(4), dtype='int64') - res = s_tft & 0 + n0 = 0 + res = s_tft & n0 expected = s_fff assert_series_equal(res, expected) - res = s_0123 & 0 + res = s_0123 & n0 expected = Series([0] * 4) assert_series_equal(res, expected) - res = s_tft & 1 + n1 = 1 + res = s_tft & n1 expected = s_tft assert_series_equal(res, expected) - res = s_0123 & 1 + res = s_0123 & n1 expected = Series([0, 1, 0, 1]) assert_series_equal(res, expected) @@ -1952,19 +1944,16 @@ def test_operators_bitwise_with_reindexing(self): # unable to sort incompatible object via .union. exp = Series([False] * 7, index=['b', 'c', 'a', 0, 1, 2, 3]) with tm.assert_produces_warning(RuntimeWarning): - result = s_tft & s_0123 - assert_series_equal(result, exp) + assert_series_equal(s_tft & s_0123, exp) else: exp = Series([False] * 7, index=[0, 1, 2, 3, 'a', 'b', 'c']) - result = s_tft & s_0123 - assert_series_equal(result, exp) + assert_series_equal(s_tft & s_0123, exp) # s_tft will be all false now because of reindexing like s_0123 if compat.PY3: # unable to sort incompatible object via .union. exp = Series([False] * 7, index=[0, 1, 2, 3, 'b', 'c', 'a']) with tm.assert_produces_warning(RuntimeWarning): - result = s_0123 & s_tft assert_series_equal(s_0123 & s_tft, exp) else: exp = Series([False] * 7, index=[0, 1, 2, 3, 'a', 'b', 'c']) @@ -1980,9 +1969,9 @@ def test_operators_bitwise_with_nans(self): s_ftft = Series([False, True, False, True]) s_abNd = Series(['a', 'b', np.NaN, 'd']) - result = s_0123 & s_abNd + res = s_0123 & s_abNd expected = s_ftft - assert_series_equal(result, expected) + assert_series_equal(res, expected) def test_scalar_na_cmp_corners(self): s = Series([2, 3, 4, 5, 6, 7, 8, 9, 10]) @@ -2193,12 +2182,12 @@ def _check_fill(meth, op, a, b, fill_value=0): with np.errstate(all='ignore'): if amask[i]: if bmask[i]: - exp_values.append(np.nan) + exp_values.append(nan) continue exp_values.append(op(fill_value, b[i])) elif bmask[i]: if amask[i]: - exp_values.append(np.nan) + exp_values.append(nan) continue exp_values.append(op(a[i], fill_value)) else: @@ -2208,8 +2197,8 @@ def _check_fill(meth, op, a, b, fill_value=0): expected = Series(exp_values, exp_index) assert_series_equal(result, expected) - a = Series([np.nan, 1., 2., 3., np.nan], index=np.arange(5)) - b = Series([np.nan, 1, np.nan, 3, np.nan, 4.], index=np.arange(6)) + a = Series([nan, 1., 2., 3., nan], index=np.arange(5)) + b = Series([nan, 1, nan, 3, nan, 4.], index=np.arange(6)) pairings = [] for op in ['add', 'sub', 'mul', 'pow', 'truediv', 'floordiv']: From 9aa41ec43c30c73da6481d639dd74aab14bf8a51 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 20 Feb 2018 17:30:58 -0800 Subject: [PATCH 5/7] fixup duplicate line --- doc/source/whatsnew/v0.23.0.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/source/whatsnew/v0.23.0.txt b/doc/source/whatsnew/v0.23.0.txt index 7e0e1fa477d50..00f9c67368e8d 100644 --- a/doc/source/whatsnew/v0.23.0.txt +++ b/doc/source/whatsnew/v0.23.0.txt @@ -589,7 +589,6 @@ Other API Changes - :func:`Series.to_csv` now accepts a ``compression`` argument that works in the same way as the ``compression`` argument in :func:`DataFrame.to_csv` (:issue:`18958`) - Set operations (union, difference...) on :class:`IntervalIndex` with incompatible index types will now raise a ``TypeError`` rather than a ``ValueError`` (:issue:`19329`) - :class:`DateOffset` objects render more simply, e.g. ```` instead of ```` (:issue:`19403`) -- :func:`pandas.merge` provides a more informative error message when trying to merge on timezone-aware and timezone-naive columns (:issue:`15800`) - Boolean operations ``&, |, ^`` between a ``Series`` and an ``Index`` will no longer raise ``TypeError`` (:issue:`19792`) .. _whatsnew_0230.deprecations: From a5910784f5182376d5743424b469e5e637252427 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 22 Feb 2018 11:23:57 -0800 Subject: [PATCH 6/7] reword comment --- pandas/core/ops.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pandas/core/ops.py b/pandas/core/ops.py index 53d5bf042dc36..011b400374a38 100644 --- a/pandas/core/ops.py +++ b/pandas/core/ops.py @@ -1199,8 +1199,7 @@ def na_op(x, y): except TypeError: assert not isinstance(y, (list, ABCSeries, pd.Index)) if isinstance(y, np.ndarray): - # This next assertion is here because there used to be - # a branch that specifically caught this case. + # bool-bool dtype operations should be OK, should not get here assert not (is_bool_dtype(x) and is_bool_dtype(y)) x = _ensure_object(x) y = _ensure_object(y) From f19a5961bb3ce313ea733873f2d8b1ae28035f38 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Fri, 23 Feb 2018 13:58:27 -0800 Subject: [PATCH 7/7] use is_list_like --- pandas/core/ops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/ops.py b/pandas/core/ops.py index 011b400374a38..ad13695441bc7 100644 --- a/pandas/core/ops.py +++ b/pandas/core/ops.py @@ -1241,7 +1241,7 @@ def wrapper(self, other): else: # scalars, list, tuple, np.array is_other_int_dtype = is_integer_dtype(np.asarray(other)) - if isinstance(other, list): + if is_list_like(other) and not isinstance(other, np.ndarray): # TODO: Can we do this before the is_integer_dtype check? # could the is_integer_dtype check be checking the wrong # thing? e.g. other = [[0, 1], [2, 3], [4, 5]]?