From 9ce186032bf565eb3542cf39ea7357f65da69d99 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Tue, 18 Sep 2018 01:39:27 +0200 Subject: [PATCH 1/7] Fixturize tests/frame/test_arithmetic --- pandas/tests/frame/conftest.py | 18 +-- pandas/tests/frame/test_arithmetic.py | 195 +++++++++++--------------- 2 files changed, 90 insertions(+), 123 deletions(-) diff --git a/pandas/tests/frame/conftest.py b/pandas/tests/frame/conftest.py index fdedb93835d75..4a4ce4540b9d5 100644 --- a/pandas/tests/frame/conftest.py +++ b/pandas/tests/frame/conftest.py @@ -70,9 +70,10 @@ def mixed_float_frame(): Columns are ['A', 'B', 'C', 'D']. """ df = DataFrame(tm.getSeriesData()) - df.A = df.A.astype('float16') + df.A = df.A.astype('float32') df.B = df.B.astype('float32') - df.C = df.C.astype('float64') + df.C = df.C.astype('float16') + df.D = df.D.astype('float64') return df @@ -84,9 +85,10 @@ def mixed_float_frame2(): Columns are ['A', 'B', 'C', 'D']. """ df = DataFrame(tm.getSeriesData()) - df.D = df.D.astype('float16') + df.D = df.D.astype('float32') df.C = df.C.astype('float32') - df.B = df.B.astype('float64') + df.B = df.B.astype('float16') + df.D = df.D.astype('float64') return df @@ -99,10 +101,10 @@ def mixed_int_frame(): """ df = DataFrame({k: v.astype(int) for k, v in compat.iteritems(tm.getSeriesData())}) - df.A = df.A.astype('uint8') - df.B = df.B.astype('int32') - df.C = df.C.astype('int64') - df.D = np.ones(len(df.D), dtype='uint64') + df.A = df.A.astype('int32') + df.B = np.ones(len(df.B), dtype='uint64') + df.C = df.C.astype('uint8') + df.D = df.C.astype('int64') return df diff --git a/pandas/tests/frame/test_arithmetic.py b/pandas/tests/frame/test_arithmetic.py index 9c61f13b944ea..a3a5a3c66b217 100644 --- a/pandas/tests/frame/test_arithmetic.py +++ b/pandas/tests/frame/test_arithmetic.py @@ -127,132 +127,97 @@ def test_df_add_flex_filled_mixed_dtypes(self): 'B': ser * 2}) tm.assert_frame_equal(result, expected) - def test_arith_flex_frame(self): - seriesd = tm.getSeriesData() - frame = pd.DataFrame(seriesd).copy() - - mixed_float = pd.DataFrame({'A': frame['A'].copy().astype('float32'), - 'B': frame['B'].copy().astype('float32'), - 'C': frame['C'].copy().astype('float16'), - 'D': frame['D'].copy().astype('float64')}) - - intframe = pd.DataFrame({k: v.astype(int) - for k, v in seriesd.items()}) - mixed_int = pd.DataFrame({'A': intframe['A'].copy().astype('int32'), - 'B': np.ones(len(intframe), dtype='uint64'), - 'C': intframe['C'].copy().astype('uint8'), - 'D': intframe['D'].copy().astype('int64')}) - - # force these all to int64 to avoid platform testing issues - intframe = pd.DataFrame({c: s for c, s in intframe.items()}, - dtype=np.int64) - - ops = ['add', 'sub', 'mul', 'div', 'truediv', 'pow', 'floordiv', 'mod'] + @pytest.mark.parametrize('op', ['add', 'sub', 'mul', 'div', 'truediv', + 'pow', 'floordiv', 'mod']) + def test_arith_flex_frame(self, op, int_frame, mixed_int_frame, + float_frame, mixed_float_frame): + if not PY3: aliases = {} else: aliases = {'div': 'truediv'} - - for op in ops: - try: - alias = aliases.get(op, op) - f = getattr(operator, alias) - result = getattr(frame, op)(2 * frame) - exp = f(frame, 2 * frame) + alias = aliases.get(op, op) + + f = getattr(operator, alias) + result = getattr(float_frame, op)(2 * float_frame) + exp = f(float_frame, 2 * float_frame) + tm.assert_frame_equal(result, exp) + + # vs mix float + result = getattr(mixed_float_frame, op)(2 * mixed_float_frame) + exp = f(mixed_float_frame, 2 * mixed_float_frame) + tm.assert_frame_equal(result, exp) + _check_mixed_float(result, dtype=dict(C=None)) + + # vs mix int + if op in ['add', 'sub', 'mul']: + result = getattr(mixed_int_frame, op)(2 + mixed_int_frame) + exp = f(mixed_int_frame, 2 + mixed_int_frame) + + # no overflow in the uint + dtype = None + if op in ['sub']: + dtype = dict(B='uint64', C=None) + elif op in ['add', 'mul']: + dtype = dict(C=None) + tm.assert_frame_equal(result, exp) + _check_mixed_int(result, dtype=dtype) + + # rops + r_f = lambda x, y: f(y, x) + result = getattr(float_frame, 'r' + op)(2 * float_frame) + exp = r_f(float_frame, 2 * float_frame) + tm.assert_frame_equal(result, exp) + + # vs mix float + result = getattr(mixed_float_frame, op)(2 * mixed_float_frame) + exp = f(mixed_float_frame, 2 * mixed_float_frame) + tm.assert_frame_equal(result, exp) + _check_mixed_float(result, dtype=dict(C=None)) + + result = getattr(int_frame, op)(2 * int_frame) + exp = f(int_frame, 2 * int_frame) + tm.assert_frame_equal(result, exp) + + # vs mix int + if op in ['add', 'sub', 'mul']: + result = getattr(mixed_int_frame, op)(2 + mixed_int_frame) + exp = f(mixed_int_frame, 2 + mixed_int_frame) + + # no overflow in the uint + dtype = None + if op in ['sub']: + dtype = dict(B='uint64', C=None) + elif op in ['add', 'mul']: + dtype = dict(C=None) tm.assert_frame_equal(result, exp) + _check_mixed_int(result, dtype=dtype) - # vs mix float - result = getattr(mixed_float, op)(2 * mixed_float) - exp = f(mixed_float, 2 * mixed_float) - tm.assert_frame_equal(result, exp) - _check_mixed_float(result, dtype=dict(C=None)) - - # vs mix int - if op in ['add', 'sub', 'mul']: - result = getattr(mixed_int, op)(2 + mixed_int) - exp = f(mixed_int, 2 + mixed_int) - - # no overflow in the uint - dtype = None - if op in ['sub']: - dtype = dict(B='uint64', C=None) - elif op in ['add', 'mul']: - dtype = dict(C=None) - tm.assert_frame_equal(result, exp) - _check_mixed_int(result, dtype=dtype) - - # rops - r_f = lambda x, y: f(y, x) - result = getattr(frame, 'r' + op)(2 * frame) - exp = r_f(frame, 2 * frame) - tm.assert_frame_equal(result, exp) - - # vs mix float - result = getattr(mixed_float, op)(2 * mixed_float) - exp = f(mixed_float, 2 * mixed_float) - tm.assert_frame_equal(result, exp) - _check_mixed_float(result, dtype=dict(C=None)) - - result = getattr(intframe, op)(2 * intframe) - exp = f(intframe, 2 * intframe) - tm.assert_frame_equal(result, exp) - - # vs mix int - if op in ['add', 'sub', 'mul']: - result = getattr(mixed_int, op)(2 + mixed_int) - exp = f(mixed_int, 2 + mixed_int) - - # no overflow in the uint - dtype = None - if op in ['sub']: - dtype = dict(B='uint64', C=None) - elif op in ['add', 'mul']: - dtype = dict(C=None) - tm.assert_frame_equal(result, exp) - _check_mixed_int(result, dtype=dtype) - except: - printing.pprint_thing("Failing operation %r" % op) - raise - - # ndim >= 3 - ndim_5 = np.ones(frame.shape + (3, 4, 5)) - msg = "Unable to coerce to Series/DataFrame" - with tm.assert_raises_regex(ValueError, msg): - f(frame, ndim_5) - - with tm.assert_raises_regex(ValueError, msg): - getattr(frame, op)(ndim_5) - - # res_add = frame.add(frame) - # res_sub = frame.sub(frame) - # res_mul = frame.mul(frame) - # res_div = frame.div(2 * frame) - - # tm.assert_frame_equal(res_add, frame + frame) - # tm.assert_frame_equal(res_sub, frame - frame) - # tm.assert_frame_equal(res_mul, frame * frame) - # tm.assert_frame_equal(res_div, frame / (2 * frame)) - - const_add = frame.add(1) - tm.assert_frame_equal(const_add, frame + 1) + # ndim >= 3 + ndim_5 = np.ones(float_frame.shape + (3, 4, 5)) + msg = "Unable to coerce to Series/DataFrame" + with tm.assert_raises_regex(ValueError, msg): + f(float_frame, ndim_5) + + with tm.assert_raises_regex(ValueError, msg): + getattr(float_frame, op)(ndim_5) + + const_add = float_frame.add(1) + tm.assert_frame_equal(const_add, float_frame + 1) # corner cases - result = frame.add(frame[:0]) - tm.assert_frame_equal(result, frame * np.nan) + result = float_frame.add(float_frame[:0]) + tm.assert_frame_equal(result, float_frame * np.nan) - result = frame[:0].add(frame) - tm.assert_frame_equal(result, frame * np.nan) + result = float_frame[:0].add(float_frame) + tm.assert_frame_equal(result, float_frame * np.nan) with tm.assert_raises_regex(NotImplementedError, 'fill_value'): - frame.add(frame.iloc[0], fill_value=3) + float_frame.add(float_frame.iloc[0], fill_value=3) with tm.assert_raises_regex(NotImplementedError, 'fill_value'): - frame.add(frame.iloc[0], axis='index', fill_value=3) - - def test_arith_flex_series(self): - arr = np.array([[1., 2., 3.], - [4., 5., 6.], - [7., 8., 9.]]) - df = pd.DataFrame(arr, columns=['one', 'two', 'three'], - index=['a', 'b', 'c']) + float_frame.add(float_frame.iloc[0], axis='index', fill_value=3) + + def test_arith_flex_series(self, simple_frame): + df = simple_frame row = df.xs('a') col = df['two'] From 0d8aa092ae21888c373d7ade6062bfd7d9c72058 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Tue, 18 Sep 2018 02:25:03 +0200 Subject: [PATCH 2/7] PEP8 fix --- pandas/tests/frame/test_arithmetic.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pandas/tests/frame/test_arithmetic.py b/pandas/tests/frame/test_arithmetic.py index a3a5a3c66b217..a69a0c46dd986 100644 --- a/pandas/tests/frame/test_arithmetic.py +++ b/pandas/tests/frame/test_arithmetic.py @@ -5,7 +5,6 @@ import numpy as np from pandas.compat import range, PY3 -import pandas.io.formats.printing as printing import pandas as pd import pandas.util.testing as tm From 1f8a5d65c294207382b8d9e62c963941aa248d8c Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Tue, 18 Sep 2018 18:49:38 +0200 Subject: [PATCH 3/7] Review (jreback) --- pandas/tests/frame/test_arithmetic.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/pandas/tests/frame/test_arithmetic.py b/pandas/tests/frame/test_arithmetic.py index a69a0c46dd986..da0cd02c20a49 100644 --- a/pandas/tests/frame/test_arithmetic.py +++ b/pandas/tests/frame/test_arithmetic.py @@ -126,18 +126,14 @@ def test_df_add_flex_filled_mixed_dtypes(self): 'B': ser * 2}) tm.assert_frame_equal(result, expected) - @pytest.mark.parametrize('op', ['add', 'sub', 'mul', 'div', 'truediv', - 'pow', 'floordiv', 'mod']) - def test_arith_flex_frame(self, op, int_frame, mixed_int_frame, - float_frame, mixed_float_frame): - - if not PY3: - aliases = {} - else: - aliases = {'div': 'truediv'} - alias = aliases.get(op, op) - - f = getattr(operator, alias) + def test_arith_flex_frame(self, all_arithmetic_operators, int_frame, + mixed_int_frame, float_frame, mixed_float_frame): + + op = all_arithmetic_operators + if op.startswith('__r'): + pytest.skip('Reverse methods not available in operator library') + + f = getattr(operator, op) result = getattr(float_frame, op)(2 * float_frame) exp = f(float_frame, 2 * float_frame) tm.assert_frame_equal(result, exp) From 1acc6b1a821ad1c18bd307dd4ea4cc170168343d Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Tue, 18 Sep 2018 22:16:01 +0200 Subject: [PATCH 4/7] PEP8 --- pandas/tests/frame/test_arithmetic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/frame/test_arithmetic.py b/pandas/tests/frame/test_arithmetic.py index da0cd02c20a49..1ca57ed35317c 100644 --- a/pandas/tests/frame/test_arithmetic.py +++ b/pandas/tests/frame/test_arithmetic.py @@ -4,7 +4,7 @@ import pytest import numpy as np -from pandas.compat import range, PY3 +from pandas.compat import range import pandas as pd import pandas.util.testing as tm From 3ee3b709f7132cbf480c14c36c4abe947ca62be9 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Thu, 20 Sep 2018 10:02:22 +0200 Subject: [PATCH 5/7] Split up test_arith_flex_frame (review jreback) --- pandas/tests/frame/test_arithmetic.py | 111 +++++++++++++------------- 1 file changed, 55 insertions(+), 56 deletions(-) diff --git a/pandas/tests/frame/test_arithmetic.py b/pandas/tests/frame/test_arithmetic.py index 1ca57ed35317c..9190635a9b1c9 100644 --- a/pandas/tests/frame/test_arithmetic.py +++ b/pandas/tests/frame/test_arithmetic.py @@ -126,14 +126,17 @@ def test_df_add_flex_filled_mixed_dtypes(self): 'B': ser * 2}) tm.assert_frame_equal(result, expected) - def test_arith_flex_frame(self, all_arithmetic_operators, int_frame, - mixed_int_frame, float_frame, mixed_float_frame): + def test_arith_flex_frame(self, all_arithmetic_operators, float_frame, + mixed_float_frame): - op = all_arithmetic_operators + op = all_arithmetic_operators # one instance of parametrized fixture if op.startswith('__r'): - pytest.skip('Reverse methods not available in operator library') + # get op without "r" and invert it + tmp = getattr(operator, op[:2] + op[3:]) + f = lambda x, y: tmp(y, x) + else: + f = getattr(operator, op) - f = getattr(operator, op) result = getattr(float_frame, op)(2 * float_frame) exp = f(float_frame, 2 * float_frame) tm.assert_frame_equal(result, exp) @@ -144,58 +147,52 @@ def test_arith_flex_frame(self, all_arithmetic_operators, int_frame, tm.assert_frame_equal(result, exp) _check_mixed_float(result, dtype=dict(C=None)) + @pytest.mark.parametrize('op', ['__add__', '__sub__', '__mul__']) + def test_arith_flex_frame_mixed(self, op, int_frame, mixed_int_frame, + float_frame, mixed_float_frame): + + if op.startswith('__r'): + # get op without "r" and invert it + tmp = getattr(operator, op[:2] + op[3:]) + f = lambda x, y: tmp(y, x) + else: + f = getattr(operator, op) + # vs mix int - if op in ['add', 'sub', 'mul']: - result = getattr(mixed_int_frame, op)(2 + mixed_int_frame) - exp = f(mixed_int_frame, 2 + mixed_int_frame) - - # no overflow in the uint - dtype = None - if op in ['sub']: - dtype = dict(B='uint64', C=None) - elif op in ['add', 'mul']: - dtype = dict(C=None) - tm.assert_frame_equal(result, exp) - _check_mixed_int(result, dtype=dtype) - - # rops - r_f = lambda x, y: f(y, x) - result = getattr(float_frame, 'r' + op)(2 * float_frame) - exp = r_f(float_frame, 2 * float_frame) - tm.assert_frame_equal(result, exp) - - # vs mix float - result = getattr(mixed_float_frame, op)(2 * mixed_float_frame) - exp = f(mixed_float_frame, 2 * mixed_float_frame) - tm.assert_frame_equal(result, exp) - _check_mixed_float(result, dtype=dict(C=None)) - - result = getattr(int_frame, op)(2 * int_frame) - exp = f(int_frame, 2 * int_frame) - tm.assert_frame_equal(result, exp) - - # vs mix int - if op in ['add', 'sub', 'mul']: - result = getattr(mixed_int_frame, op)(2 + mixed_int_frame) - exp = f(mixed_int_frame, 2 + mixed_int_frame) - - # no overflow in the uint - dtype = None - if op in ['sub']: - dtype = dict(B='uint64', C=None) - elif op in ['add', 'mul']: - dtype = dict(C=None) - tm.assert_frame_equal(result, exp) - _check_mixed_int(result, dtype=dtype) - - # ndim >= 3 - ndim_5 = np.ones(float_frame.shape + (3, 4, 5)) - msg = "Unable to coerce to Series/DataFrame" - with tm.assert_raises_regex(ValueError, msg): - f(float_frame, ndim_5) - - with tm.assert_raises_regex(ValueError, msg): - getattr(float_frame, op)(ndim_5) + result = getattr(mixed_int_frame, op)(2 + mixed_int_frame) + exp = f(mixed_int_frame, 2 + mixed_int_frame) + + # no overflow in the uint + dtype = None + if op in ['__sub__']: + dtype = dict(B='uint64', C=None) + elif op in ['__add__', '__mul__']: + dtype = dict(C=None) + tm.assert_frame_equal(result, exp) + _check_mixed_int(result, dtype=dtype) + + # vs mix float + result = getattr(mixed_float_frame, op)(2 * mixed_float_frame) + exp = f(mixed_float_frame, 2 * mixed_float_frame) + tm.assert_frame_equal(result, exp) + _check_mixed_float(result, dtype=dict(C=None)) + + # vs plain int + result = getattr(int_frame, op)(2 * int_frame) + exp = f(int_frame, 2 * int_frame) + tm.assert_frame_equal(result, exp) + + def test_arith_flex_frame_corner(self, all_arithmetic_operators, + float_frame): + + op = all_arithmetic_operators + + # Check that arrays with dim >= 3 raise + for dim in range(3, 6): + arr = np.ones((1,) * dim) + msg = "Unable to coerce to Series/DataFrame" + with tm.assert_raises_regex(ValueError, msg): + getattr(float_frame, op)(arr) const_add = float_frame.add(1) tm.assert_frame_equal(const_add, float_frame + 1) @@ -206,8 +203,10 @@ def test_arith_flex_frame(self, all_arithmetic_operators, int_frame, result = float_frame[:0].add(float_frame) tm.assert_frame_equal(result, float_frame * np.nan) + with tm.assert_raises_regex(NotImplementedError, 'fill_value'): float_frame.add(float_frame.iloc[0], fill_value=3) + with tm.assert_raises_regex(NotImplementedError, 'fill_value'): float_frame.add(float_frame.iloc[0], axis='index', fill_value=3) From b6e14b09455f8f11ea370d661fd5cfa6e4118dc6 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Sun, 23 Sep 2018 15:06:40 +0200 Subject: [PATCH 6/7] Review (jreback) --- pandas/tests/frame/test_arithmetic.py | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/pandas/tests/frame/test_arithmetic.py b/pandas/tests/frame/test_arithmetic.py index 9190635a9b1c9..1da208b2ec627 100644 --- a/pandas/tests/frame/test_arithmetic.py +++ b/pandas/tests/frame/test_arithmetic.py @@ -128,14 +128,14 @@ def test_df_add_flex_filled_mixed_dtypes(self): def test_arith_flex_frame(self, all_arithmetic_operators, float_frame, mixed_float_frame): + # one instance of parametrized fixture + op = all_arithmetic_operators - op = all_arithmetic_operators # one instance of parametrized fixture - if op.startswith('__r'): - # get op without "r" and invert it - tmp = getattr(operator, op[:2] + op[3:]) - f = lambda x, y: tmp(y, x) - else: - f = getattr(operator, op) + def f(x, y): + if op.startswith('__r'): + # get op without "r" and invert it + return getattr(operator, op.replace('__r', '__'))(y, x) + return getattr(operator, op)(x, y) result = getattr(float_frame, op)(2 * float_frame) exp = f(float_frame, 2 * float_frame) @@ -149,14 +149,8 @@ def test_arith_flex_frame(self, all_arithmetic_operators, float_frame, @pytest.mark.parametrize('op', ['__add__', '__sub__', '__mul__']) def test_arith_flex_frame_mixed(self, op, int_frame, mixed_int_frame, - float_frame, mixed_float_frame): - - if op.startswith('__r'): - # get op without "r" and invert it - tmp = getattr(operator, op[:2] + op[3:]) - f = lambda x, y: tmp(y, x) - else: - f = getattr(operator, op) + mixed_float_frame): + f = getattr(operator, op) # vs mix int result = getattr(mixed_int_frame, op)(2 + mixed_int_frame) @@ -184,7 +178,7 @@ def test_arith_flex_frame_mixed(self, op, int_frame, mixed_int_frame, def test_arith_flex_frame_corner(self, all_arithmetic_operators, float_frame): - + # one instance of parametrized fixture op = all_arithmetic_operators # Check that arrays with dim >= 3 raise From b0b82a7cbcf1cf47cb660cb3e8f7ebe0427bc63d Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Wed, 26 Sep 2018 08:04:45 +0200 Subject: [PATCH 7/7] Review (jbrockmendel) --- pandas/tests/frame/test_arithmetic.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pandas/tests/frame/test_arithmetic.py b/pandas/tests/frame/test_arithmetic.py index 1da208b2ec627..2b08897864db0 100644 --- a/pandas/tests/frame/test_arithmetic.py +++ b/pandas/tests/frame/test_arithmetic.py @@ -132,8 +132,8 @@ def test_arith_flex_frame(self, all_arithmetic_operators, float_frame, op = all_arithmetic_operators def f(x, y): + # r-versions not in operator-stdlib; get op without "r" and invert if op.startswith('__r'): - # get op without "r" and invert it return getattr(operator, op.replace('__r', '__'))(y, x) return getattr(operator, op)(x, y) @@ -176,8 +176,8 @@ def test_arith_flex_frame_mixed(self, op, int_frame, mixed_int_frame, exp = f(int_frame, 2 * int_frame) tm.assert_frame_equal(result, exp) - def test_arith_flex_frame_corner(self, all_arithmetic_operators, - float_frame): + def test_arith_flex_frame_raise(self, all_arithmetic_operators, + float_frame): # one instance of parametrized fixture op = all_arithmetic_operators @@ -188,6 +188,8 @@ def test_arith_flex_frame_corner(self, all_arithmetic_operators, with tm.assert_raises_regex(ValueError, msg): getattr(float_frame, op)(arr) + def test_arith_flex_frame_corner(self, float_frame): + const_add = float_frame.add(1) tm.assert_frame_equal(const_add, float_frame + 1)