From 61bdf3f2577aac1639a3ec2e1c18c8a1f7d7c0a1 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Thu, 19 Sep 2019 14:28:11 -0700 Subject: [PATCH 01/16] CLN: stop catching BaseException --- pandas/tests/test_nanops.py | 101 ++++++++++++------------------------ 1 file changed, 33 insertions(+), 68 deletions(-) diff --git a/pandas/tests/test_nanops.py b/pandas/tests/test_nanops.py index eb39f01657b90..9c9e49124fc75 100644 --- a/pandas/tests/test_nanops.py +++ b/pandas/tests/test_nanops.py @@ -165,25 +165,17 @@ def check_fun_data( else: targ = targfunc(targartempval, axis=axis, **kwargs) - try: - res = testfunc(testarval, axis=axis, skipna=skipna, **kwargs) + res = testfunc(testarval, axis=axis, skipna=skipna, **kwargs) + self.check_results(targ, res, axis, check_dtype=check_dtype) + if skipna: + res = testfunc(testarval, axis=axis, **kwargs) + self.check_results(targ, res, axis, check_dtype=check_dtype) + if axis is None: + res = testfunc(testarval, skipna=skipna, **kwargs) + self.check_results(targ, res, axis, check_dtype=check_dtype) + if skipna and axis is None: + res = testfunc(testarval, **kwargs) self.check_results(targ, res, axis, check_dtype=check_dtype) - if skipna: - res = testfunc(testarval, axis=axis, **kwargs) - self.check_results(targ, res, axis, check_dtype=check_dtype) - if axis is None: - res = testfunc(testarval, skipna=skipna, **kwargs) - self.check_results(targ, res, axis, check_dtype=check_dtype) - if skipna and axis is None: - res = testfunc(testarval, **kwargs) - self.check_results(targ, res, axis, check_dtype=check_dtype) - except BaseException as exc: - exc.args += ( - "axis: {axis} of {of}".format(axis=axis, of=testarval.ndim - 1), - "skipna: {skipna}".format(skipna=skipna), - "kwargs: {kwargs}".format(kwargs=kwargs), - ) - raise if testarval.ndim <= 1: return @@ -222,23 +214,15 @@ def check_fun( testarval = getattr(self, testar) targarval = getattr(self, targar) targarnanval = getattr(self, targarnan) - try: - self.check_fun_data( - testfunc, - targfunc, - testarval, - targarval, - targarnanval, - empty_targfunc=empty_targfunc, - **kwargs - ) - except BaseException as exc: - exc.args += ( - "testar: {testar}".format(testar=testar), - "targar: {targar}".format(targar=targar), - "targarnan: {targarnan}".format(targarnan=targarnan), - ) - raise + self.check_fun_data( + testfunc, + targfunc, + testarval, + targarval, + targarnanval, + empty_targfunc=empty_targfunc, + **kwargs + ) def check_funs( self, @@ -753,15 +737,12 @@ def test_nanne(self): def check_bool(self, func, value, correct, *args, **kwargs): while getattr(value, "ndim", True): - try: - res0 = func(value, *args, **kwargs) - if correct: - assert res0 - else: - assert not res0 - except BaseException as exc: - exc.args += ("dim: {}".format(getattr(value, "ndim", value)),) - raise + res0 = func(value, *args, **kwargs) + if correct: + assert res0 + else: + assert not res0 + if not hasattr(value, "ndim"): break try: @@ -796,21 +777,13 @@ def test__has_infs(self): for arr, correct in pairs: val = getattr(self, arr) - try: - self.check_bool(nanops._has_infs, val, correct) - except BaseException as exc: - exc.args += (arr,) - raise + self.check_bool(nanops._has_infs, val, correct) for arr, correct in pairs_float: val = getattr(self, arr) - try: - self.check_bool(nanops._has_infs, val, correct) - self.check_bool(nanops._has_infs, val.astype("f4"), correct) - self.check_bool(nanops._has_infs, val.astype("f2"), correct) - except BaseException as exc: - exc.args += (arr,) - raise + self.check_bool(nanops._has_infs, val, correct) + self.check_bool(nanops._has_infs, val.astype("f4"), correct) + self.check_bool(nanops._has_infs, val.astype("f2"), correct) def test__isfinite(self): pairs = [ @@ -844,21 +817,13 @@ def test__isfinite(self): for arr, correct in pairs: val = getattr(self, arr) - try: - self.check_bool(func1, val, correct) - except BaseException as exc: - exc.args += (arr,) - raise + self.check_bool(func1, val, correct) for arr, correct in pairs_float: val = getattr(self, arr) - try: - self.check_bool(func1, val, correct) - self.check_bool(func1, val.astype("f4"), correct) - self.check_bool(func1, val.astype("f2"), correct) - except BaseException as exc: - exc.args += (arr,) - raise + self.check_bool(func1, val, correct) + self.check_bool(func1, val.astype("f4"), correct) + self.check_bool(func1, val.astype("f2"), correct) def test__bn_ok_dtype(self): assert nanops._bn_ok_dtype(self.arr_float.dtype, "test") From b2ac45d1280419cc6d6270240c1475fb97e62f86 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Thu, 19 Sep 2019 14:35:03 -0700 Subject: [PATCH 02/16] CLN: Exception in nanops --- pandas/tests/test_nanops.py | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/pandas/tests/test_nanops.py b/pandas/tests/test_nanops.py index 9c9e49124fc75..41b27f030d80f 100644 --- a/pandas/tests/test_nanops.py +++ b/pandas/tests/test_nanops.py @@ -681,23 +681,19 @@ def check_nancomp(self, checkfun, targ0): arr_nan_float1 = self.arr_nan_float1 while targ0.ndim: - try: - res0 = checkfun(arr_float, arr_float1) - tm.assert_almost_equal(targ0, res0) + res0 = checkfun(arr_float, arr_float1) + tm.assert_almost_equal(targ0, res0) - if targ0.ndim > 1: - targ1 = np.vstack([targ0, arr_nan]) - else: - targ1 = np.hstack([targ0, arr_nan]) - res1 = checkfun(arr_float_nan, arr_float1_nan) - tm.assert_numpy_array_equal(targ1, res1, check_dtype=False) - - targ2 = arr_nan_nan - res2 = checkfun(arr_float_nan, arr_nan_float1) - tm.assert_numpy_array_equal(targ2, res2, check_dtype=False) - except Exception as exc: - exc.args += ("ndim: {arr_float.ndim}".format(arr_float=arr_float),) - raise + if targ0.ndim > 1: + targ1 = np.vstack([targ0, arr_nan]) + else: + targ1 = np.hstack([targ0, arr_nan]) + res1 = checkfun(arr_float_nan, arr_float1_nan) + tm.assert_numpy_array_equal(targ1, res1, check_dtype=False) + + targ2 = arr_nan_nan + res2 = checkfun(arr_float_nan, arr_nan_float1) + tm.assert_numpy_array_equal(targ2, res2, check_dtype=False) try: arr_float = np.take(arr_float, 0, axis=-1) From 576032c676fd9dfbc543a1af4bc5bade4eec3058 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Thu, 19 Sep 2019 14:53:34 -0700 Subject: [PATCH 03/16] parametrize --- pandas/tests/test_nanops.py | 55 +++++++++++++------------------------ 1 file changed, 19 insertions(+), 36 deletions(-) diff --git a/pandas/tests/test_nanops.py b/pandas/tests/test_nanops.py index 41b27f030d80f..132e9d50d4e0f 100644 --- a/pandas/tests/test_nanops.py +++ b/pandas/tests/test_nanops.py @@ -1,4 +1,5 @@ from functools import partial +import operator import warnings import numpy as np @@ -300,20 +301,14 @@ def _badobj_wrap(self, value, func, allow_complex=True, **kwargs): value = value.astype("f8") return func(value, **kwargs) - def test_nanany(self): + @pytest.mark.parametrize("npop,nanop", [ + (np.any, nanops.nanany), + (np.all, nanops.nanall) + ]) + def test_nan_bool_reductions(self, npop, nanop): self.check_funs( - nanops.nanany, - np.any, - allow_all_nan=False, - allow_str=False, - allow_date=False, - allow_tdelta=False, - ) - - def test_nanall(self): - self.check_funs( - nanops.nanall, - np.all, + nanop, + npop, allow_all_nan=False, allow_str=False, allow_date=False, @@ -707,29 +702,17 @@ def check_nancomp(self, checkfun, targ0): except ValueError: break - def test_nangt(self): - targ0 = self.arr_float > self.arr_float1 - self.check_nancomp(nanops.nangt, targ0) - - def test_nange(self): - targ0 = self.arr_float >= self.arr_float1 - self.check_nancomp(nanops.nange, targ0) - - def test_nanlt(self): - targ0 = self.arr_float < self.arr_float1 - self.check_nancomp(nanops.nanlt, targ0) - - def test_nanle(self): - targ0 = self.arr_float <= self.arr_float1 - self.check_nancomp(nanops.nanle, targ0) - - def test_naneq(self): - targ0 = self.arr_float == self.arr_float1 - self.check_nancomp(nanops.naneq, targ0) - - def test_nanne(self): - targ0 = self.arr_float != self.arr_float1 - self.check_nancomp(nanops.nanne, targ0) + @pytest.mark.parametrize("op,nanop", [ + (operator.eq, nanops.naneq), + (operator.ne, nanops.nanne), + (operator.gt, nanops.nangt), + (operator.ge, nanops.nange), + (operator.lt, nanops.nanlt), + (operator.le, nanops.nanle), + ]) + def test_nan_comparison(self, op, nanop): + targ0 = op(self.arr_float, self.arr_float1) + self.check_nancomp(nanop, targ0) def check_bool(self, func, value, correct, *args, **kwargs): while getattr(value, "ndim", True): From e47b58da48f89a43bdd1a501ff1409d9e4a087d5 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Thu, 19 Sep 2019 15:36:26 -0700 Subject: [PATCH 04/16] checkpoint --- pandas/tests/test_nanops.py | 169 +++++++++++++++++++++++------------- 1 file changed, 107 insertions(+), 62 deletions(-) diff --git a/pandas/tests/test_nanops.py b/pandas/tests/test_nanops.py index 132e9d50d4e0f..87c92f66bcb05 100644 --- a/pandas/tests/test_nanops.py +++ b/pandas/tests/test_nanops.py @@ -186,6 +186,7 @@ def check_fun_data( targarval2 = np.take(targarval, 0, axis=-1) targarnanval2 = np.take(targarnanval, 0, axis=-1) except ValueError: + raise # See if this is hit return self.check_fun_data( testfunc, @@ -237,6 +238,7 @@ def check_funs( allow_obj=True, **kwargs ): + # TODO: do we *ever* allow_str? self.check_fun(testfunc, targfunc, "arr_float", **kwargs) self.check_fun(testfunc, targfunc, "arr_float_nan", "arr_float", **kwargs) self.check_fun(testfunc, targfunc, "arr_int", **kwargs) @@ -268,6 +270,7 @@ def check_funs( try: targfunc(self.arr_date) except TypeError: + raise # see if this is hit pass else: self.check_fun(testfunc, targfunc, "arr_date", **kwargs) @@ -277,6 +280,7 @@ def check_funs( try: targfunc(self.arr_tdelta) except TypeError: + # FIXME: this is hit, but shouldnt be (looks like #28289 doesnt fix) pass else: self.check_fun(testfunc, targfunc, "arr_tdelta", **kwargs) @@ -301,27 +305,31 @@ def _badobj_wrap(self, value, func, allow_complex=True, **kwargs): value = value.astype("f8") return func(value, **kwargs) - @pytest.mark.parametrize("npop,nanop", [ - (np.any, nanops.nanany), - (np.all, nanops.nanall) - ]) + @pytest.mark.parametrize( + "npop,nanop", [(np.any, nanops.nanany), (np.all, nanops.nanall)] + ) def test_nan_bool_reductions(self, npop, nanop): self.check_funs( nanop, npop, + allow_complex=True, allow_all_nan=False, allow_str=False, allow_date=False, allow_tdelta=False, + allow_obj=True, ) def test_nansum(self): self.check_funs( nanops.nansum, np.sum, + allow_complex=True, + allow_all_nan=True, allow_str=False, allow_date=False, allow_tdelta=True, + allow_obj=True, check_dtype=False, empty_targfunc=np.nansum, ) @@ -331,10 +339,11 @@ def test_nanmean(self): nanops.nanmean, np.mean, allow_complex=False, - allow_obj=False, + allow_all_nan=True, allow_str=False, - allow_date=False, + allow_date=False, # FIXME: this should be allowed allow_tdelta=True, + allow_obj=False, ) def test_nanmean_overflow(self): @@ -350,22 +359,31 @@ def test_nanmean_overflow(self): assert result == np_result assert result.dtype == np.float64 - def test_returned_dtype(self): - - dtypes = [np.int16, np.int32, np.int64, np.float32, np.float64] - if hasattr(np, "float128"): - dtypes.append(np.float128) + @pytest.mark.parametrize( + "dtype", + [ + np.int16, + np.int32, + np.int64, + np.float32, + np.float64, + getattr(np, "float128", None), + ], + ) + def test_returned_dtype(self, dtype): + if dtype is None: + # No float128 available + return - for dtype in dtypes: - s = Series(range(10), dtype=dtype) - group_a = ["mean", "std", "var", "skew", "kurt"] - group_b = ["min", "max"] - for method in group_a + group_b: - result = getattr(s, method)() - if is_integer_dtype(dtype) and method in group_a: - assert result.dtype == np.float64 - else: - assert result.dtype == dtype + ser = Series(range(10), dtype=dtype) + group_a = ["mean", "std", "var", "skew", "kurt"] + group_b = ["min", "max"] + for method in group_a + group_b: + result = getattr(ser, method)() + if is_integer_dtype(dtype) and method in group_a: + assert result.dtype == np.float64 + else: + assert result.dtype == dtype def test_nanmedian(self): with warnings.catch_warnings(record=True): @@ -374,34 +392,31 @@ def test_nanmedian(self): nanops.nanmedian, np.median, allow_complex=False, + allow_all_nan=True, allow_str=False, allow_date=False, allow_tdelta=True, allow_obj="convert", ) + @pytest.mark.parametrize( + "nanop,npop,allow_tdelta", + [ + (nanops.nanvar, np.var, True), # FIXME: #28289 should disallow tdelta + (nanops.nanstd, np.std, True), + # TODO: nice way of getting test_nansem in here? + ], + ) @pytest.mark.parametrize("ddof", range(3)) - def test_nanvar(self, ddof): + def test_ddof_nanop(self, ddof, nanop, npop, allow_tdelta): self.check_funs( - nanops.nanvar, - np.var, - allow_complex=False, - allow_str=False, - allow_date=False, - allow_tdelta=True, - allow_obj="convert", - ddof=ddof, - ) - - @pytest.mark.parametrize("ddof", range(3)) - def test_nanstd(self, ddof): - self.check_funs( - nanops.nanstd, - np.std, + nanop, + npop, allow_complex=False, + allow_all_nan=True, allow_str=False, allow_date=False, - allow_tdelta=True, + allow_tdelta=allow_tdelta, allow_obj="convert", ddof=ddof, ) @@ -416,6 +431,7 @@ def test_nansem(self, ddof): nanops.nansem, sem, allow_complex=False, + allow_all_nan=True, allow_str=False, allow_date=False, allow_tdelta=False, @@ -423,25 +439,33 @@ def test_nansem(self, ddof): ddof=ddof, ) - def _minmax_wrap(self, value, axis=None, func=None): - - # numpy warns if all nan - res = func(value, axis) - if res.dtype.kind == "m": - res = np.atleast_1d(res) - return res - def test_nanmin(self): with warnings.catch_warnings(record=True): warnings.simplefilter("ignore", RuntimeWarning) - func = partial(self._minmax_wrap, func=np.min) - self.check_funs(nanops.nanmin, func, allow_str=False, allow_obj=False) + self.check_funs( + nanops.nanmin, + np.min, + allow_complex=True, + allow_all_nan=True, + allow_str=False, + allow_date=True, + allow_tdelta=True, + allow_obj=False, + ) def test_nanmax(self): with warnings.catch_warnings(): warnings.simplefilter("ignore", RuntimeWarning) - func = partial(self._minmax_wrap, func=np.max) - self.check_funs(nanops.nanmax, func, allow_str=False, allow_obj=False) + self.check_funs( + nanops.nanmax, + np.max, + allow_complex=True, + allow_all_nan=True, + allow_str=False, + allow_date=True, + allow_tdelta=True, + allow_obj=False, + ) def _argminmax_wrap(self, value, axis=None, func=None): res = func(value, axis) @@ -465,17 +489,28 @@ def test_nanargmax(self): self.check_funs( nanops.nanargmax, func, + allow_complex=True, + allow_all_nan=True, allow_str=False, - allow_obj=False, allow_date=True, allow_tdelta=True, + allow_obj=False, ) def test_nanargmin(self): with warnings.catch_warnings(record=True): warnings.simplefilter("ignore", RuntimeWarning) func = partial(self._argminmax_wrap, func=np.argmin) - self.check_funs(nanops.nanargmin, func, allow_str=False, allow_obj=False) + self.check_funs( + nanops.nanargmin, + func, + allow_complex=True, + allow_all_nan=True, + allow_str=False, + allow_date=True, + allow_tdelta=True, + allow_obj=False, + ) def _skew_kurt_wrap(self, values, axis=None, func=None): if not isinstance(values.dtype.type, np.floating): @@ -499,9 +534,11 @@ def test_nanskew(self): nanops.nanskew, func, allow_complex=False, + allow_all_nan=True, allow_str=False, allow_date=False, allow_tdelta=False, + allow_obj=True, ) @td.skip_if_no_scipy @@ -515,18 +552,23 @@ def test_nankurt(self): nanops.nankurt, func, allow_complex=False, + allow_all_nan=True, allow_str=False, allow_date=False, allow_tdelta=False, + allow_obj=True, ) def test_nanprod(self): self.check_funs( nanops.nanprod, np.prod, + allow_complex=True, + allow_all_nan=True, allow_str=False, allow_date=False, allow_tdelta=False, + allow_obj=True, empty_targfunc=np.nanprod, ) @@ -702,21 +744,24 @@ def check_nancomp(self, checkfun, targ0): except ValueError: break - @pytest.mark.parametrize("op,nanop", [ - (operator.eq, nanops.naneq), - (operator.ne, nanops.nanne), - (operator.gt, nanops.nangt), - (operator.ge, nanops.nange), - (operator.lt, nanops.nanlt), - (operator.le, nanops.nanle), - ]) + @pytest.mark.parametrize( + "op,nanop", + [ + (operator.eq, nanops.naneq), + (operator.ne, nanops.nanne), + (operator.gt, nanops.nangt), + (operator.ge, nanops.nange), + (operator.lt, nanops.nanlt), + (operator.le, nanops.nanle), + ], + ) def test_nan_comparison(self, op, nanop): targ0 = op(self.arr_float, self.arr_float1) self.check_nancomp(nanop, targ0) - def check_bool(self, func, value, correct, *args, **kwargs): + def check_bool(self, func, value, correct): while getattr(value, "ndim", True): - res0 = func(value, *args, **kwargs) + res0 = func(value) if correct: assert res0 else: From de32a8f0637ceb19e18eedc925511be7ce078968 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Thu, 19 Sep 2019 15:38:17 -0700 Subject: [PATCH 05/16] parametrize --- pandas/tests/test_nanops.py | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/pandas/tests/test_nanops.py b/pandas/tests/test_nanops.py index 87c92f66bcb05..37cf16f8d5111 100644 --- a/pandas/tests/test_nanops.py +++ b/pandas/tests/test_nanops.py @@ -439,26 +439,16 @@ def test_nansem(self, ddof): ddof=ddof, ) - def test_nanmin(self): + @pytest.mark.parametrize("nanop,npop", [ + (nanops.nanmin, np.min), + (nanops.nanmax, np.max), + ]) + def test_nanminmax(self, nanop, npop): with warnings.catch_warnings(record=True): warnings.simplefilter("ignore", RuntimeWarning) self.check_funs( - nanops.nanmin, - np.min, - allow_complex=True, - allow_all_nan=True, - allow_str=False, - allow_date=True, - allow_tdelta=True, - allow_obj=False, - ) - - def test_nanmax(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", RuntimeWarning) - self.check_funs( - nanops.nanmax, - np.max, + nanop, + npop, allow_complex=True, allow_all_nan=True, allow_str=False, From 4ee11b9cefce5c53359d92f653087d6a91025ca5 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Thu, 19 Sep 2019 15:51:25 -0700 Subject: [PATCH 06/16] checkpoint --- pandas/tests/test_nanops.py | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/pandas/tests/test_nanops.py b/pandas/tests/test_nanops.py index 37cf16f8d5111..5bb6aabd8c7c9 100644 --- a/pandas/tests/test_nanops.py +++ b/pandas/tests/test_nanops.py @@ -181,13 +181,10 @@ def check_fun_data( if testarval.ndim <= 1: return - try: - testarval2 = np.take(testarval, 0, axis=-1) - targarval2 = np.take(targarval, 0, axis=-1) - targarnanval2 = np.take(targarnanval, 0, axis=-1) - except ValueError: - raise # See if this is hit - return + testarval2 = np.take(testarval, 0, axis=-1) + targarval2 = np.take(targarval, 0, axis=-1) + targarnanval2 = np.take(targarnanval, 0, axis=-1) + self.check_fun_data( testfunc, targfunc, @@ -209,6 +206,7 @@ def check_fun( empty_targfunc=None, **kwargs ): + assert targarnan is None, targarnan if targar is None: targar = testar if targarnan is None: @@ -267,14 +265,9 @@ def check_funs( objs += [self.arr_str.astype("O"), self.arr_utf.astype("O")] if allow_date: - try: - targfunc(self.arr_date) - except TypeError: - raise # see if this is hit - pass - else: - self.check_fun(testfunc, targfunc, "arr_date", **kwargs) - objs += [self.arr_date.astype("O")] + targfunc(self.arr_date) + self.check_fun(testfunc, targfunc, "arr_date", **kwargs) + objs += [self.arr_date.astype("O")] if allow_tdelta: try: From 66e47e80c382c835ff5f526f3318cadf9ff428a2 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Thu, 19 Sep 2019 16:55:31 -0700 Subject: [PATCH 07/16] checkpoint --- pandas/tests/test_nanops.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/pandas/tests/test_nanops.py b/pandas/tests/test_nanops.py index 5bb6aabd8c7c9..5fedabd5d3fbe 100644 --- a/pandas/tests/test_nanops.py +++ b/pandas/tests/test_nanops.py @@ -153,11 +153,12 @@ def check_fun_data( targfunc, testarval, targarval, - targarnanval, check_dtype=True, empty_targfunc=None, **kwargs ): + targarnanval = testarval + for axis in list(range(targarval.ndim)) + [None]: for skipna in [False, True]: targartempval = targarval if skipna else targarnanval @@ -183,14 +184,12 @@ def check_fun_data( testarval2 = np.take(testarval, 0, axis=-1) targarval2 = np.take(targarval, 0, axis=-1) - targarnanval2 = np.take(targarnanval, 0, axis=-1) self.check_fun_data( testfunc, targfunc, testarval2, targarval2, - targarnanval2, check_dtype=check_dtype, empty_targfunc=empty_targfunc, **kwargs @@ -202,24 +201,24 @@ def check_fun( targfunc, testar, targar=None, - targarnan=None, - empty_targfunc=None, **kwargs ): - assert targarnan is None, targarnan + + empty_targfunc = None + if testfunc.__name__ not in ["nanargmin", "nanargmax"]: + # TODO: why not these two? + empty_targfunc = getattr(np, testfunc.__name__, None) + + if targar is None: targar = testar - if targarnan is None: - targarnan = testar testarval = getattr(self, testar) targarval = getattr(self, targar) - targarnanval = getattr(self, targarnan) self.check_fun_data( testfunc, targfunc, testarval, targarval, - targarnanval, empty_targfunc=empty_targfunc, **kwargs ) @@ -238,7 +237,7 @@ def check_funs( ): # TODO: do we *ever* allow_str? self.check_fun(testfunc, targfunc, "arr_float", **kwargs) - self.check_fun(testfunc, targfunc, "arr_float_nan", "arr_float", **kwargs) + self.check_fun(testfunc, targfunc, "arr_float_nan", targar="arr_float", **kwargs) self.check_fun(testfunc, targfunc, "arr_int", **kwargs) self.check_fun(testfunc, targfunc, "arr_bool", **kwargs) objs = [ @@ -324,7 +323,6 @@ def test_nansum(self): allow_tdelta=True, allow_obj=True, check_dtype=False, - empty_targfunc=np.nansum, ) def test_nanmean(self): @@ -552,7 +550,6 @@ def test_nanprod(self): allow_date=False, allow_tdelta=False, allow_obj=True, - empty_targfunc=np.nanprod, ) def check_nancorr_nancov_2d(self, checkfun, targ0, targ1, **kwargs): From 1800f1f0da1aaa4444eeadac423d98d0022c62fc Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Fri, 20 Sep 2019 10:59:51 -0700 Subject: [PATCH 08/16] revert --- pandas/tests/test_nanops.py | 240 +++++++++++++++++------------------- 1 file changed, 116 insertions(+), 124 deletions(-) diff --git a/pandas/tests/test_nanops.py b/pandas/tests/test_nanops.py index 5fedabd5d3fbe..41b27f030d80f 100644 --- a/pandas/tests/test_nanops.py +++ b/pandas/tests/test_nanops.py @@ -1,5 +1,4 @@ from functools import partial -import operator import warnings import numpy as np @@ -153,12 +152,11 @@ def check_fun_data( targfunc, testarval, targarval, + targarnanval, check_dtype=True, empty_targfunc=None, **kwargs ): - targarnanval = testarval - for axis in list(range(targarval.ndim)) + [None]: for skipna in [False, True]: targartempval = targarval if skipna else targarnanval @@ -182,14 +180,18 @@ def check_fun_data( if testarval.ndim <= 1: return - testarval2 = np.take(testarval, 0, axis=-1) - targarval2 = np.take(targarval, 0, axis=-1) - + try: + testarval2 = np.take(testarval, 0, axis=-1) + targarval2 = np.take(targarval, 0, axis=-1) + targarnanval2 = np.take(targarnanval, 0, axis=-1) + except ValueError: + return self.check_fun_data( testfunc, targfunc, testarval2, targarval2, + targarnanval2, check_dtype=check_dtype, empty_targfunc=empty_targfunc, **kwargs @@ -201,24 +203,23 @@ def check_fun( targfunc, testar, targar=None, + targarnan=None, + empty_targfunc=None, **kwargs ): - - empty_targfunc = None - if testfunc.__name__ not in ["nanargmin", "nanargmax"]: - # TODO: why not these two? - empty_targfunc = getattr(np, testfunc.__name__, None) - - if targar is None: targar = testar + if targarnan is None: + targarnan = testar testarval = getattr(self, testar) targarval = getattr(self, targar) + targarnanval = getattr(self, targarnan) self.check_fun_data( testfunc, targfunc, testarval, targarval, + targarnanval, empty_targfunc=empty_targfunc, **kwargs ) @@ -235,9 +236,8 @@ def check_funs( allow_obj=True, **kwargs ): - # TODO: do we *ever* allow_str? self.check_fun(testfunc, targfunc, "arr_float", **kwargs) - self.check_fun(testfunc, targfunc, "arr_float_nan", targar="arr_float", **kwargs) + self.check_fun(testfunc, targfunc, "arr_float_nan", "arr_float", **kwargs) self.check_fun(testfunc, targfunc, "arr_int", **kwargs) self.check_fun(testfunc, targfunc, "arr_bool", **kwargs) objs = [ @@ -264,15 +264,18 @@ def check_funs( objs += [self.arr_str.astype("O"), self.arr_utf.astype("O")] if allow_date: - targfunc(self.arr_date) - self.check_fun(testfunc, targfunc, "arr_date", **kwargs) - objs += [self.arr_date.astype("O")] + try: + targfunc(self.arr_date) + except TypeError: + pass + else: + self.check_fun(testfunc, targfunc, "arr_date", **kwargs) + objs += [self.arr_date.astype("O")] if allow_tdelta: try: targfunc(self.arr_tdelta) except TypeError: - # FIXME: this is hit, but shouldnt be (looks like #28289 doesnt fix) pass else: self.check_fun(testfunc, targfunc, "arr_tdelta", **kwargs) @@ -297,32 +300,35 @@ def _badobj_wrap(self, value, func, allow_complex=True, **kwargs): value = value.astype("f8") return func(value, **kwargs) - @pytest.mark.parametrize( - "npop,nanop", [(np.any, nanops.nanany), (np.all, nanops.nanall)] - ) - def test_nan_bool_reductions(self, npop, nanop): + def test_nanany(self): + self.check_funs( + nanops.nanany, + np.any, + allow_all_nan=False, + allow_str=False, + allow_date=False, + allow_tdelta=False, + ) + + def test_nanall(self): self.check_funs( - nanop, - npop, - allow_complex=True, + nanops.nanall, + np.all, allow_all_nan=False, allow_str=False, allow_date=False, allow_tdelta=False, - allow_obj=True, ) def test_nansum(self): self.check_funs( nanops.nansum, np.sum, - allow_complex=True, - allow_all_nan=True, allow_str=False, allow_date=False, allow_tdelta=True, - allow_obj=True, check_dtype=False, + empty_targfunc=np.nansum, ) def test_nanmean(self): @@ -330,11 +336,10 @@ def test_nanmean(self): nanops.nanmean, np.mean, allow_complex=False, - allow_all_nan=True, + allow_obj=False, allow_str=False, - allow_date=False, # FIXME: this should be allowed + allow_date=False, allow_tdelta=True, - allow_obj=False, ) def test_nanmean_overflow(self): @@ -350,31 +355,22 @@ def test_nanmean_overflow(self): assert result == np_result assert result.dtype == np.float64 - @pytest.mark.parametrize( - "dtype", - [ - np.int16, - np.int32, - np.int64, - np.float32, - np.float64, - getattr(np, "float128", None), - ], - ) - def test_returned_dtype(self, dtype): - if dtype is None: - # No float128 available - return + def test_returned_dtype(self): - ser = Series(range(10), dtype=dtype) - group_a = ["mean", "std", "var", "skew", "kurt"] - group_b = ["min", "max"] - for method in group_a + group_b: - result = getattr(ser, method)() - if is_integer_dtype(dtype) and method in group_a: - assert result.dtype == np.float64 - else: - assert result.dtype == dtype + dtypes = [np.int16, np.int32, np.int64, np.float32, np.float64] + if hasattr(np, "float128"): + dtypes.append(np.float128) + + for dtype in dtypes: + s = Series(range(10), dtype=dtype) + group_a = ["mean", "std", "var", "skew", "kurt"] + group_b = ["min", "max"] + for method in group_a + group_b: + result = getattr(s, method)() + if is_integer_dtype(dtype) and method in group_a: + assert result.dtype == np.float64 + else: + assert result.dtype == dtype def test_nanmedian(self): with warnings.catch_warnings(record=True): @@ -383,31 +379,34 @@ def test_nanmedian(self): nanops.nanmedian, np.median, allow_complex=False, - allow_all_nan=True, allow_str=False, allow_date=False, allow_tdelta=True, allow_obj="convert", ) - @pytest.mark.parametrize( - "nanop,npop,allow_tdelta", - [ - (nanops.nanvar, np.var, True), # FIXME: #28289 should disallow tdelta - (nanops.nanstd, np.std, True), - # TODO: nice way of getting test_nansem in here? - ], - ) @pytest.mark.parametrize("ddof", range(3)) - def test_ddof_nanop(self, ddof, nanop, npop, allow_tdelta): + def test_nanvar(self, ddof): + self.check_funs( + nanops.nanvar, + np.var, + allow_complex=False, + allow_str=False, + allow_date=False, + allow_tdelta=True, + allow_obj="convert", + ddof=ddof, + ) + + @pytest.mark.parametrize("ddof", range(3)) + def test_nanstd(self, ddof): self.check_funs( - nanop, - npop, + nanops.nanstd, + np.std, allow_complex=False, - allow_all_nan=True, allow_str=False, allow_date=False, - allow_tdelta=allow_tdelta, + allow_tdelta=True, allow_obj="convert", ddof=ddof, ) @@ -422,7 +421,6 @@ def test_nansem(self, ddof): nanops.nansem, sem, allow_complex=False, - allow_all_nan=True, allow_str=False, allow_date=False, allow_tdelta=False, @@ -430,23 +428,25 @@ def test_nansem(self, ddof): ddof=ddof, ) - @pytest.mark.parametrize("nanop,npop", [ - (nanops.nanmin, np.min), - (nanops.nanmax, np.max), - ]) - def test_nanminmax(self, nanop, npop): + def _minmax_wrap(self, value, axis=None, func=None): + + # numpy warns if all nan + res = func(value, axis) + if res.dtype.kind == "m": + res = np.atleast_1d(res) + return res + + def test_nanmin(self): with warnings.catch_warnings(record=True): warnings.simplefilter("ignore", RuntimeWarning) - self.check_funs( - nanop, - npop, - allow_complex=True, - allow_all_nan=True, - allow_str=False, - allow_date=True, - allow_tdelta=True, - allow_obj=False, - ) + func = partial(self._minmax_wrap, func=np.min) + self.check_funs(nanops.nanmin, func, allow_str=False, allow_obj=False) + + def test_nanmax(self): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", RuntimeWarning) + func = partial(self._minmax_wrap, func=np.max) + self.check_funs(nanops.nanmax, func, allow_str=False, allow_obj=False) def _argminmax_wrap(self, value, axis=None, func=None): res = func(value, axis) @@ -470,28 +470,17 @@ def test_nanargmax(self): self.check_funs( nanops.nanargmax, func, - allow_complex=True, - allow_all_nan=True, allow_str=False, + allow_obj=False, allow_date=True, allow_tdelta=True, - allow_obj=False, ) def test_nanargmin(self): with warnings.catch_warnings(record=True): warnings.simplefilter("ignore", RuntimeWarning) func = partial(self._argminmax_wrap, func=np.argmin) - self.check_funs( - nanops.nanargmin, - func, - allow_complex=True, - allow_all_nan=True, - allow_str=False, - allow_date=True, - allow_tdelta=True, - allow_obj=False, - ) + self.check_funs(nanops.nanargmin, func, allow_str=False, allow_obj=False) def _skew_kurt_wrap(self, values, axis=None, func=None): if not isinstance(values.dtype.type, np.floating): @@ -515,11 +504,9 @@ def test_nanskew(self): nanops.nanskew, func, allow_complex=False, - allow_all_nan=True, allow_str=False, allow_date=False, allow_tdelta=False, - allow_obj=True, ) @td.skip_if_no_scipy @@ -533,23 +520,19 @@ def test_nankurt(self): nanops.nankurt, func, allow_complex=False, - allow_all_nan=True, allow_str=False, allow_date=False, allow_tdelta=False, - allow_obj=True, ) def test_nanprod(self): self.check_funs( nanops.nanprod, np.prod, - allow_complex=True, - allow_all_nan=True, allow_str=False, allow_date=False, allow_tdelta=False, - allow_obj=True, + empty_targfunc=np.nanprod, ) def check_nancorr_nancov_2d(self, checkfun, targ0, targ1, **kwargs): @@ -724,24 +707,33 @@ def check_nancomp(self, checkfun, targ0): except ValueError: break - @pytest.mark.parametrize( - "op,nanop", - [ - (operator.eq, nanops.naneq), - (operator.ne, nanops.nanne), - (operator.gt, nanops.nangt), - (operator.ge, nanops.nange), - (operator.lt, nanops.nanlt), - (operator.le, nanops.nanle), - ], - ) - def test_nan_comparison(self, op, nanop): - targ0 = op(self.arr_float, self.arr_float1) - self.check_nancomp(nanop, targ0) - - def check_bool(self, func, value, correct): + def test_nangt(self): + targ0 = self.arr_float > self.arr_float1 + self.check_nancomp(nanops.nangt, targ0) + + def test_nange(self): + targ0 = self.arr_float >= self.arr_float1 + self.check_nancomp(nanops.nange, targ0) + + def test_nanlt(self): + targ0 = self.arr_float < self.arr_float1 + self.check_nancomp(nanops.nanlt, targ0) + + def test_nanle(self): + targ0 = self.arr_float <= self.arr_float1 + self.check_nancomp(nanops.nanle, targ0) + + def test_naneq(self): + targ0 = self.arr_float == self.arr_float1 + self.check_nancomp(nanops.naneq, targ0) + + def test_nanne(self): + targ0 = self.arr_float != self.arr_float1 + self.check_nancomp(nanops.nanne, targ0) + + def check_bool(self, func, value, correct, *args, **kwargs): while getattr(value, "ndim", True): - res0 = func(value) + res0 = func(value, *args, **kwargs) if correct: assert res0 else: From 778d6542f5b1aff7a69b3e01ffe314caa4be73dc Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Fri, 20 Sep 2019 11:05:36 -0700 Subject: [PATCH 09/16] TST: parametrize, catch less --- pandas/tests/test_nanops.py | 99 +++++++++++++++---------------------- 1 file changed, 39 insertions(+), 60 deletions(-) diff --git a/pandas/tests/test_nanops.py b/pandas/tests/test_nanops.py index 41b27f030d80f..c9915b9352c30 100644 --- a/pandas/tests/test_nanops.py +++ b/pandas/tests/test_nanops.py @@ -1,4 +1,5 @@ from functools import partial +import operator import warnings import numpy as np @@ -15,6 +16,7 @@ import pandas.util.testing as tm use_bn = nanops._USE_BOTTLENECK +has_c16 = hasattr(np, "complex128") class TestnanopsDataFrame: @@ -131,14 +133,9 @@ def _coerce_tds(targ, res): if targ.dtype.kind != "O": res = res.astype(targ.dtype) else: - try: - res = res.astype("c16") - except RuntimeError: - res = res.astype("f8") - try: - targ = targ.astype("c16") - except RuntimeError: - targ = targ.astype("f8") + cast_dtype = "c16" if has_c16 else "f8" + res = res.astype(cast_dtype) + targ = targ.astype(cast_dtype) # there should never be a case where numpy returns an object # but nanops doesn't, so make that an exception elif targ.dtype.kind == "O": @@ -180,12 +177,10 @@ def check_fun_data( if testarval.ndim <= 1: return - try: - testarval2 = np.take(testarval, 0, axis=-1) - targarval2 = np.take(targarval, 0, axis=-1) - targarnanval2 = np.take(targarnanval, 0, axis=-1) - except ValueError: - return + # Recurse on lower-dimension + testarval2 = np.take(testarval, 0, axis=-1) + targarval2 = np.take(targarval, 0, axis=-1) + targarnanval2 = np.take(targarnanval, 0, axis=-1) self.check_fun_data( testfunc, targfunc, @@ -264,13 +259,9 @@ def check_funs( objs += [self.arr_str.astype("O"), self.arr_utf.astype("O")] if allow_date: - try: - targfunc(self.arr_date) - except TypeError: - pass - else: - self.check_fun(testfunc, targfunc, "arr_date", **kwargs) - objs += [self.arr_date.astype("O")] + targfunc(self.arr_date) + self.check_fun(testfunc, targfunc, "arr_date", **kwargs) + objs += [self.arr_date.astype("O")] if allow_tdelta: try: @@ -695,41 +686,30 @@ def check_nancomp(self, checkfun, targ0): res2 = checkfun(arr_float_nan, arr_nan_float1) tm.assert_numpy_array_equal(targ2, res2, check_dtype=False) - try: - arr_float = np.take(arr_float, 0, axis=-1) - arr_float1 = np.take(arr_float1, 0, axis=-1) - arr_nan = np.take(arr_nan, 0, axis=-1) - arr_nan_nan = np.take(arr_nan_nan, 0, axis=-1) - arr_float_nan = np.take(arr_float_nan, 0, axis=-1) - arr_float1_nan = np.take(arr_float1_nan, 0, axis=-1) - arr_nan_float1 = np.take(arr_nan_float1, 0, axis=-1) - targ0 = np.take(targ0, 0, axis=-1) - except ValueError: - break - - def test_nangt(self): - targ0 = self.arr_float > self.arr_float1 - self.check_nancomp(nanops.nangt, targ0) - - def test_nange(self): - targ0 = self.arr_float >= self.arr_float1 - self.check_nancomp(nanops.nange, targ0) - - def test_nanlt(self): - targ0 = self.arr_float < self.arr_float1 - self.check_nancomp(nanops.nanlt, targ0) - - def test_nanle(self): - targ0 = self.arr_float <= self.arr_float1 - self.check_nancomp(nanops.nanle, targ0) - - def test_naneq(self): - targ0 = self.arr_float == self.arr_float1 - self.check_nancomp(nanops.naneq, targ0) - - def test_nanne(self): - targ0 = self.arr_float != self.arr_float1 - self.check_nancomp(nanops.nanne, targ0) + # Lower dimension for next step in the loop + arr_float = np.take(arr_float, 0, axis=-1) + arr_float1 = np.take(arr_float1, 0, axis=-1) + arr_nan = np.take(arr_nan, 0, axis=-1) + arr_nan_nan = np.take(arr_nan_nan, 0, axis=-1) + arr_float_nan = np.take(arr_float_nan, 0, axis=-1) + arr_float1_nan = np.take(arr_float1_nan, 0, axis=-1) + arr_nan_float1 = np.take(arr_nan_float1, 0, axis=-1) + targ0 = np.take(targ0, 0, axis=-1) + + @pytest.mark.parametrize( + "op,nanop", + [ + (operator.eq, nanops.naneq), + (operator.ne, nanops.nanne), + (operator.gt, nanops.nangt), + (operator.ge, nanops.nange), + (operator.lt, nanops.nanlt), + (operator.le, nanops.nanle), + ], + ) + def test_nan_comparison(self, op, nanop): + targ0 = op(self.arr_float, self.arr_float1) + self.check_nancomp(nanop, targ0) def check_bool(self, func, value, correct, *args, **kwargs): while getattr(value, "ndim", True): @@ -741,10 +721,9 @@ def check_bool(self, func, value, correct, *args, **kwargs): if not hasattr(value, "ndim"): break - try: - value = np.take(value, 0, axis=-1) - except ValueError: - break + + # Reduce dimension for next step in the loop + value = np.take(value, 0, axis=-1) def test__has_infs(self): pairs = [ From 1cd77b4a8bde66b158dee1ccd50ee32dd4fe445e Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Fri, 20 Sep 2019 11:09:04 -0700 Subject: [PATCH 10/16] parametrize --- pandas/tests/test_nanops.py | 41 ++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/pandas/tests/test_nanops.py b/pandas/tests/test_nanops.py index c9915b9352c30..8cfa35d149722 100644 --- a/pandas/tests/test_nanops.py +++ b/pandas/tests/test_nanops.py @@ -346,22 +346,31 @@ def test_nanmean_overflow(self): assert result == np_result assert result.dtype == np.float64 - def test_returned_dtype(self): - - dtypes = [np.int16, np.int32, np.int64, np.float32, np.float64] - if hasattr(np, "float128"): - dtypes.append(np.float128) - - for dtype in dtypes: - s = Series(range(10), dtype=dtype) - group_a = ["mean", "std", "var", "skew", "kurt"] - group_b = ["min", "max"] - for method in group_a + group_b: - result = getattr(s, method)() - if is_integer_dtype(dtype) and method in group_a: - assert result.dtype == np.float64 - else: - assert result.dtype == dtype + @pytest.mark.parametrize( + "dtype", + [ + np.int16, + np.int32, + np.int64, + np.float32, + np.float64, + getattr(np, "float128", None), + ], + ) + def test_returned_dtype(self, dtype): + if dtype is None: + # no float128 available + return + + s = Series(range(10), dtype=dtype) + group_a = ["mean", "std", "var", "skew", "kurt"] + group_b = ["min", "max"] + for method in group_a + group_b: + result = getattr(s, method)() + if is_integer_dtype(dtype) and method in group_a: + assert result.dtype == np.float64 + else: + assert result.dtype == dtype def test_nanmedian(self): with warnings.catch_warnings(record=True): From 7fc298cc599a7dcdbc2d8ac07fb11b1cba5cc3ee Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Fri, 20 Sep 2019 11:14:36 -0700 Subject: [PATCH 11/16] CLN: remove always-False condition, redundant arg --- pandas/tests/test_nanops.py | 44 ++++++------------------------------- 1 file changed, 7 insertions(+), 37 deletions(-) diff --git a/pandas/tests/test_nanops.py b/pandas/tests/test_nanops.py index 8cfa35d149722..2ecf01e1e8535 100644 --- a/pandas/tests/test_nanops.py +++ b/pandas/tests/test_nanops.py @@ -149,14 +149,13 @@ def check_fun_data( targfunc, testarval, targarval, - targarnanval, check_dtype=True, empty_targfunc=None, **kwargs ): for axis in list(range(targarval.ndim)) + [None]: for skipna in [False, True]: - targartempval = targarval if skipna else targarnanval + targartempval = targarval if skipna else testarval if skipna and empty_targfunc and isna(targartempval).all(): targ = empty_targfunc(targartempval, axis=axis, **kwargs) else: @@ -180,41 +179,30 @@ def check_fun_data( # Recurse on lower-dimension testarval2 = np.take(testarval, 0, axis=-1) targarval2 = np.take(targarval, 0, axis=-1) - targarnanval2 = np.take(targarnanval, 0, axis=-1) self.check_fun_data( testfunc, targfunc, testarval2, targarval2, - targarnanval2, check_dtype=check_dtype, empty_targfunc=empty_targfunc, **kwargs ) def check_fun( - self, - testfunc, - targfunc, - testar, - targar=None, - targarnan=None, - empty_targfunc=None, - **kwargs + self, testfunc, targfunc, testar, targar=None, empty_targfunc=None, **kwargs ): + if targar is None: targar = testar - if targarnan is None: - targarnan = testar + testarval = getattr(self, testar) targarval = getattr(self, targar) - targarnanval = getattr(self, targarnan) self.check_fun_data( testfunc, targfunc, testarval, targarval, - targarnanval, empty_targfunc=empty_targfunc, **kwargs ) @@ -225,7 +213,6 @@ def check_funs( targfunc, allow_complex=True, allow_all_nan=True, - allow_str=True, allow_date=True, allow_tdelta=True, allow_obj=True, @@ -253,11 +240,6 @@ def check_funs( self.check_fun(testfunc, targfunc, "arr_nan_nanj", **kwargs) objs += [self.arr_complex.astype("O")] - if allow_str: - self.check_fun(testfunc, targfunc, "arr_str", **kwargs) - self.check_fun(testfunc, targfunc, "arr_utf", **kwargs) - objs += [self.arr_str.astype("O"), self.arr_utf.astype("O")] - if allow_date: targfunc(self.arr_date) self.check_fun(testfunc, targfunc, "arr_date", **kwargs) @@ -296,7 +278,6 @@ def test_nanany(self): nanops.nanany, np.any, allow_all_nan=False, - allow_str=False, allow_date=False, allow_tdelta=False, ) @@ -306,7 +287,6 @@ def test_nanall(self): nanops.nanall, np.all, allow_all_nan=False, - allow_str=False, allow_date=False, allow_tdelta=False, ) @@ -315,7 +295,6 @@ def test_nansum(self): self.check_funs( nanops.nansum, np.sum, - allow_str=False, allow_date=False, allow_tdelta=True, check_dtype=False, @@ -328,7 +307,6 @@ def test_nanmean(self): np.mean, allow_complex=False, allow_obj=False, - allow_str=False, allow_date=False, allow_tdelta=True, ) @@ -379,7 +357,6 @@ def test_nanmedian(self): nanops.nanmedian, np.median, allow_complex=False, - allow_str=False, allow_date=False, allow_tdelta=True, allow_obj="convert", @@ -391,7 +368,6 @@ def test_nanvar(self, ddof): nanops.nanvar, np.var, allow_complex=False, - allow_str=False, allow_date=False, allow_tdelta=True, allow_obj="convert", @@ -404,7 +380,6 @@ def test_nanstd(self, ddof): nanops.nanstd, np.std, allow_complex=False, - allow_str=False, allow_date=False, allow_tdelta=True, allow_obj="convert", @@ -421,7 +396,6 @@ def test_nansem(self, ddof): nanops.nansem, sem, allow_complex=False, - allow_str=False, allow_date=False, allow_tdelta=False, allow_obj="convert", @@ -440,13 +414,13 @@ def test_nanmin(self): with warnings.catch_warnings(record=True): warnings.simplefilter("ignore", RuntimeWarning) func = partial(self._minmax_wrap, func=np.min) - self.check_funs(nanops.nanmin, func, allow_str=False, allow_obj=False) + self.check_funs(nanops.nanmin, func, allow_obj=False) def test_nanmax(self): with warnings.catch_warnings(): warnings.simplefilter("ignore", RuntimeWarning) func = partial(self._minmax_wrap, func=np.max) - self.check_funs(nanops.nanmax, func, allow_str=False, allow_obj=False) + self.check_funs(nanops.nanmax, func, allow_obj=False) def _argminmax_wrap(self, value, axis=None, func=None): res = func(value, axis) @@ -470,7 +444,6 @@ def test_nanargmax(self): self.check_funs( nanops.nanargmax, func, - allow_str=False, allow_obj=False, allow_date=True, allow_tdelta=True, @@ -480,7 +453,7 @@ def test_nanargmin(self): with warnings.catch_warnings(record=True): warnings.simplefilter("ignore", RuntimeWarning) func = partial(self._argminmax_wrap, func=np.argmin) - self.check_funs(nanops.nanargmin, func, allow_str=False, allow_obj=False) + self.check_funs(nanops.nanargmin, func, allow_obj=False) def _skew_kurt_wrap(self, values, axis=None, func=None): if not isinstance(values.dtype.type, np.floating): @@ -504,7 +477,6 @@ def test_nanskew(self): nanops.nanskew, func, allow_complex=False, - allow_str=False, allow_date=False, allow_tdelta=False, ) @@ -520,7 +492,6 @@ def test_nankurt(self): nanops.nankurt, func, allow_complex=False, - allow_str=False, allow_date=False, allow_tdelta=False, ) @@ -529,7 +500,6 @@ def test_nanprod(self): self.check_funs( nanops.nanprod, np.prod, - allow_str=False, allow_date=False, allow_tdelta=False, empty_targfunc=np.nanprod, From 4dda70a50e95b56ab14b12d7b24e8d0e4f43a76c Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Fri, 20 Sep 2019 11:36:55 -0700 Subject: [PATCH 12/16] CLN: unnecessary args, parametrize --- pandas/tests/test_nanops.py | 74 ++++++++++--------------------------- 1 file changed, 19 insertions(+), 55 deletions(-) diff --git a/pandas/tests/test_nanops.py b/pandas/tests/test_nanops.py index 2ecf01e1e8535..c428fde8c2aa3 100644 --- a/pandas/tests/test_nanops.py +++ b/pandas/tests/test_nanops.py @@ -189,12 +189,11 @@ def check_fun_data( **kwargs ) - def check_fun( - self, testfunc, targfunc, testar, targar=None, empty_targfunc=None, **kwargs - ): + def check_fun(self, testfunc, targfunc, testar, empty_targfunc=None, **kwargs): - if targar is None: - targar = testar + targar = testar + if testar.endswith("_nan") and hasattr(self, testar[:-4]): + targar = testar[:-4] testarval = getattr(self, testar) targarval = getattr(self, targar) @@ -219,7 +218,7 @@ def check_funs( **kwargs ): self.check_fun(testfunc, targfunc, "arr_float", **kwargs) - self.check_fun(testfunc, targfunc, "arr_float_nan", "arr_float", **kwargs) + self.check_fun(testfunc, targfunc, "arr_float_nan", **kwargs) self.check_fun(testfunc, targfunc, "arr_int", **kwargs) self.check_fun(testfunc, targfunc, "arr_bool", **kwargs) objs = [ @@ -233,9 +232,7 @@ def check_funs( if allow_complex: self.check_fun(testfunc, targfunc, "arr_complex", **kwargs) - self.check_fun( - testfunc, targfunc, "arr_complex_nan", "arr_complex", **kwargs - ) + self.check_fun(testfunc, targfunc, "arr_complex_nan", **kwargs) if allow_all_nan: self.check_fun(testfunc, targfunc, "arr_nan_nanj", **kwargs) objs += [self.arr_complex.astype("O")] @@ -273,22 +270,12 @@ def _badobj_wrap(self, value, func, allow_complex=True, **kwargs): value = value.astype("f8") return func(value, **kwargs) - def test_nanany(self): - self.check_funs( - nanops.nanany, - np.any, - allow_all_nan=False, - allow_date=False, - allow_tdelta=False, - ) - - def test_nanall(self): + @pytest.mark.parametrize( + "nan_op, np_op", [(nanops.nanany, np.any), (nanops.nanall, np.all)] + ) + def test_nan_funcs(self, nan_op, np_op): self.check_funs( - nanops.nanall, - np.all, - allow_all_nan=False, - allow_date=False, - allow_tdelta=False, + nan_op, np_op, allow_all_nan=False, allow_date=False, allow_tdelta=False ) def test_nansum(self): @@ -296,7 +283,6 @@ def test_nansum(self): nanops.nansum, np.sum, allow_date=False, - allow_tdelta=True, check_dtype=False, empty_targfunc=np.nansum, ) @@ -308,7 +294,6 @@ def test_nanmean(self): allow_complex=False, allow_obj=False, allow_date=False, - allow_tdelta=True, ) def test_nanmean_overflow(self): @@ -358,7 +343,6 @@ def test_nanmedian(self): np.median, allow_complex=False, allow_date=False, - allow_tdelta=True, allow_obj="convert", ) @@ -369,7 +353,6 @@ def test_nanvar(self, ddof): np.var, allow_complex=False, allow_date=False, - allow_tdelta=True, allow_obj="convert", ddof=ddof, ) @@ -381,7 +364,6 @@ def test_nanstd(self, ddof): np.std, allow_complex=False, allow_date=False, - allow_tdelta=True, allow_obj="convert", ddof=ddof, ) @@ -402,25 +384,13 @@ def test_nansem(self, ddof): ddof=ddof, ) - def _minmax_wrap(self, value, axis=None, func=None): - - # numpy warns if all nan - res = func(value, axis) - if res.dtype.kind == "m": - res = np.atleast_1d(res) - return res - - def test_nanmin(self): + @pytest.mark.parametrize( + "nan_op,np_op", [(nanops.nanmin, np.min), (nanops.nanmax, np.max)] + ) + def test_nanops_with_warnings(self, nan_op, np_op): with warnings.catch_warnings(record=True): warnings.simplefilter("ignore", RuntimeWarning) - func = partial(self._minmax_wrap, func=np.min) - self.check_funs(nanops.nanmin, func, allow_obj=False) - - def test_nanmax(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", RuntimeWarning) - func = partial(self._minmax_wrap, func=np.max) - self.check_funs(nanops.nanmax, func, allow_obj=False) + self.check_funs(nan_op, np_op, allow_obj=False) def _argminmax_wrap(self, value, axis=None, func=None): res = func(value, axis) @@ -441,13 +411,7 @@ def test_nanargmax(self): with warnings.catch_warnings(record=True): warnings.simplefilter("ignore", RuntimeWarning) func = partial(self._argminmax_wrap, func=np.argmax) - self.check_funs( - nanops.nanargmax, - func, - allow_obj=False, - allow_date=True, - allow_tdelta=True, - ) + self.check_funs(nanops.nanargmax, func, allow_obj=False) def test_nanargmin(self): with warnings.catch_warnings(record=True): @@ -690,9 +654,9 @@ def test_nan_comparison(self, op, nanop): targ0 = op(self.arr_float, self.arr_float1) self.check_nancomp(nanop, targ0) - def check_bool(self, func, value, correct, *args, **kwargs): + def check_bool(self, func, value, correct): while getattr(value, "ndim", True): - res0 = func(value, *args, **kwargs) + res0 = func(value) if correct: assert res0 else: From 2d1ab83183007b3ae802148bf15f32c40af0888a Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Fri, 20 Sep 2019 11:52:17 -0700 Subject: [PATCH 13/16] cleanup, comments --- pandas/tests/test_nanops.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pandas/tests/test_nanops.py b/pandas/tests/test_nanops.py index c428fde8c2aa3..49d1777df0751 100644 --- a/pandas/tests/test_nanops.py +++ b/pandas/tests/test_nanops.py @@ -271,9 +271,10 @@ def _badobj_wrap(self, value, func, allow_complex=True, **kwargs): return func(value, **kwargs) @pytest.mark.parametrize( - "nan_op, np_op", [(nanops.nanany, np.any), (nanops.nanall, np.all)] + "nan_op,np_op", [(nanops.nanany, np.any), (nanops.nanall, np.all)] ) def test_nan_funcs(self, nan_op, np_op): + # TODO: allow tdelta, doesn't break tests self.check_funs( nan_op, np_op, allow_all_nan=False, allow_date=False, allow_tdelta=False ) @@ -291,7 +292,7 @@ def test_nanmean(self): self.check_funs( nanops.nanmean, np.mean, - allow_complex=False, + allow_complex=False, # TODO: allow this, doesn't break test allow_obj=False, allow_date=False, ) From dc89f2da152b7d5c740ef87060e3388d245f6de5 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Fri, 20 Sep 2019 11:59:14 -0700 Subject: [PATCH 14/16] parametrize --- pandas/tests/test_expressions.py | 86 ++++++++++++++++---------------- 1 file changed, 42 insertions(+), 44 deletions(-) diff --git a/pandas/tests/test_expressions.py b/pandas/tests/test_expressions.py index b11698bf89cda..284d2b0b5af7e 100644 --- a/pandas/tests/test_expressions.py +++ b/pandas/tests/test_expressions.py @@ -167,29 +167,29 @@ def test_invalid(self): "opname,op_str", [("add", "+"), ("sub", "-"), ("mul", "*"), ("truediv", "/"), ("pow", "**")], ) - def test_binary_ops(self, opname, op_str): + @pytest.mark.parametrize("left,right", [(_frame, _frame2), (_mixed, _mixed2)]) + def test_binary_ops(self, opname, op_str, left, right): def testit(): - for f, f2 in [(self.frame, self.frame2), (self.mixed, self.mixed2)]: + if opname == "pow": + # TODO: get this working + return - if opname == "pow": - continue + op = getattr(operator, opname) - op = getattr(operator, opname) + result = expr._can_use_numexpr(op, op_str, left, left, "evaluate") + assert result != left._is_mixed_type - result = expr._can_use_numexpr(op, op_str, f, f, "evaluate") - assert result != f._is_mixed_type + result = expr.evaluate(op, op_str, left, left, use_numexpr=True) + expected = expr.evaluate(op, op_str, left, left, use_numexpr=False) - result = expr.evaluate(op, op_str, f, f, use_numexpr=True) - expected = expr.evaluate(op, op_str, f, f, use_numexpr=False) + if isinstance(result, DataFrame): + tm.assert_frame_equal(result, expected) + else: + tm.assert_numpy_array_equal(result, expected.values) - if isinstance(result, DataFrame): - tm.assert_frame_equal(result, expected) - else: - tm.assert_numpy_array_equal(result, expected.values) - - result = expr._can_use_numexpr(op, op_str, f2, f2, "evaluate") - assert not result + result = expr._can_use_numexpr(op, op_str, right, right, "evaluate") + assert not result expr.set_use_numexpr(False) testit() @@ -210,30 +210,29 @@ def testit(): ("ne", "!="), ], ) - def test_comparison_ops(self, opname, op_str): + @pytest.mark.parametrize("left,right", [(_frame, _frame2), (_mixed, _mixed2)]) + def test_comparison_ops(self, opname, op_str, left, right): def testit(): - for f, f2 in [(self.frame, self.frame2), (self.mixed, self.mixed2)]: + f11 = left + f12 = left + 1 - f11 = f - f12 = f + 1 + f21 = right + f22 = right + 1 - f21 = f2 - f22 = f2 + 1 + op = getattr(operator, opname) - op = getattr(operator, opname) + result = expr._can_use_numexpr(op, op_str, f11, f12, "evaluate") + assert result != f11._is_mixed_type - result = expr._can_use_numexpr(op, op_str, f11, f12, "evaluate") - assert result != f11._is_mixed_type + result = expr.evaluate(op, op_str, f11, f12, use_numexpr=True) + expected = expr.evaluate(op, op_str, f11, f12, use_numexpr=False) + if isinstance(result, DataFrame): + tm.assert_frame_equal(result, expected) + else: + tm.assert_numpy_array_equal(result, expected.values) - result = expr.evaluate(op, op_str, f11, f12, use_numexpr=True) - expected = expr.evaluate(op, op_str, f11, f12, use_numexpr=False) - if isinstance(result, DataFrame): - tm.assert_frame_equal(result, expected) - else: - tm.assert_numpy_array_equal(result, expected.values) - - result = expr._can_use_numexpr(op, op_str, f21, f22, "evaluate") - assert not result + result = expr._can_use_numexpr(op, op_str, f21, f22, "evaluate") + assert not result expr.set_use_numexpr(False) testit() @@ -244,15 +243,14 @@ def testit(): testit() @pytest.mark.parametrize("cond", [True, False]) - def test_where(self, cond): + @pytest.mark.parametrize("df", [_frame, _frame2, _mixed, _mixed2]) + def test_where(self, cond, df): def testit(): - for f in [self.frame, self.frame2, self.mixed, self.mixed2]: - - c = np.empty(f.shape, dtype=np.bool_) - c.fill(cond) - result = expr.where(c, f.values, f.values + 1) - expected = np.where(c, f.values, f.values + 1) - tm.assert_numpy_array_equal(result, expected) + c = np.empty(df.shape, dtype=np.bool_) + c.fill(cond) + result = expr.where(c, df.values, df.values + 1) + expected = np.where(c, df.values, df.values + 1) + tm.assert_numpy_array_equal(result, expected) expr.set_use_numexpr(False) testit() @@ -263,7 +261,7 @@ def testit(): testit() @pytest.mark.parametrize( - "op_str,opname", list(zip(["/", "//", "**"], ["truediv", "floordiv", "pow"])) + "op_str,opname", [("/", "truediv"), ("//", "floordiv"), ("**", "pow")] ) def test_bool_ops_raise_on_arithmetic(self, op_str, opname): df = DataFrame({"a": np.random.rand(10) > 0.5, "b": np.random.rand(10) > 0.5}) @@ -291,7 +289,7 @@ def test_bool_ops_raise_on_arithmetic(self, op_str, opname): f(df, True) @pytest.mark.parametrize( - "op_str,opname", list(zip(["+", "*", "-"], ["add", "mul", "sub"])) + "op_str,opname", [("+", "add"), ("*", "mul"), ("-", "sub")] ) def test_bool_ops_warn_on_arithmetic(self, op_str, opname): n = 10 From 0503c8e86c4a6a514f121d36a085f5aeb2609111 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Fri, 20 Sep 2019 12:03:45 -0700 Subject: [PATCH 15/16] CLN --- pandas/tests/test_expressions.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/pandas/tests/test_expressions.py b/pandas/tests/test_expressions.py index 284d2b0b5af7e..62c7f098a8217 100644 --- a/pandas/tests/test_expressions.py +++ b/pandas/tests/test_expressions.py @@ -213,25 +213,22 @@ def testit(): @pytest.mark.parametrize("left,right", [(_frame, _frame2), (_mixed, _mixed2)]) def test_comparison_ops(self, opname, op_str, left, right): def testit(): - f11 = left f12 = left + 1 - - f21 = right f22 = right + 1 op = getattr(operator, opname) - result = expr._can_use_numexpr(op, op_str, f11, f12, "evaluate") - assert result != f11._is_mixed_type + result = expr._can_use_numexpr(op, op_str, left, f12, "evaluate") + assert result != left._is_mixed_type - result = expr.evaluate(op, op_str, f11, f12, use_numexpr=True) - expected = expr.evaluate(op, op_str, f11, f12, use_numexpr=False) + result = expr.evaluate(op, op_str, left, f12, use_numexpr=True) + expected = expr.evaluate(op, op_str, left, f12, use_numexpr=False) if isinstance(result, DataFrame): tm.assert_frame_equal(result, expected) else: tm.assert_numpy_array_equal(result, expected.values) - result = expr._can_use_numexpr(op, op_str, f21, f22, "evaluate") + result = expr._can_use_numexpr(op, op_str, right, f22, "evaluate") assert not result expr.set_use_numexpr(False) From 73bb58fb63c0811bf79c69e8b3863d1cb38604fa Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Fri, 20 Sep 2019 12:06:51 -0700 Subject: [PATCH 16/16] CLN --- pandas/tests/test_expressions.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/pandas/tests/test_expressions.py b/pandas/tests/test_expressions.py index 62c7f098a8217..6edd3125331b9 100644 --- a/pandas/tests/test_expressions.py +++ b/pandas/tests/test_expressions.py @@ -54,14 +54,12 @@ def run_arithmetic(self, df, other): operations = ["add", "sub", "mul", "mod", "truediv", "floordiv"] for test_flex in [True, False]: for arith in operations: - - operator_name = arith - + # TODO: share with run_binary if test_flex: op = lambda x, y: getattr(x, arith)(y) op.__name__ = arith else: - op = getattr(operator, operator_name) + op = getattr(operator, arith) expr.set_use_numexpr(False) expected = op(df, other) expr.set_use_numexpr(True) @@ -87,13 +85,14 @@ def run_binary(self, df, other): for test_flex in [True, False]: for arith in operations: if test_flex: - op = lambda x, y: getattr(df, arith)(y) + op = lambda x, y: getattr(x, arith)(y) op.__name__ = arith else: op = getattr(operator, arith) expr.set_use_numexpr(False) expected = op(df, other) expr.set_use_numexpr(True) + expr.get_test_result() result = op(df, other) used_numexpr = expr.get_test_result()