From 6d7834d973505ccdc0543242ba9eab8916fa0994 Mon Sep 17 00:00:00 2001 From: Anjana S Date: Wed, 19 Dec 2018 22:35:57 +0530 Subject: [PATCH 01/17] closes #24353 --- pandas/core/computation/ops.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/core/computation/ops.py b/pandas/core/computation/ops.py index cbdb3525d5e88..c03bad1c0faa9 100644 --- a/pandas/core/computation/ops.py +++ b/pandas/core/computation/ops.py @@ -23,7 +23,8 @@ _unary_math_ops = ('sin', 'cos', 'exp', 'log', 'expm1', 'log1p', 'sqrt', 'sinh', 'cosh', 'tanh', 'arcsin', 'arccos', - 'arctan', 'arccosh', 'arcsinh', 'arctanh', 'abs', 'log10') + 'arctan', 'arccosh', 'arcsinh', 'arctanh', 'abs', 'log10', + 'floor', 'ceil') _binary_math_ops = ('arctan2',) _mathops = _unary_math_ops + _binary_math_ops From e92bd075905cfc10df14651f9bd0d1192a337baf Mon Sep 17 00:00:00 2001 From: Anjana S Date: Sun, 23 Dec 2018 17:58:55 +0530 Subject: [PATCH 02/17] update whatsnew --- doc/source/whatsnew/v0.24.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v0.24.0.rst b/doc/source/whatsnew/v0.24.0.rst index a84fd118061bc..f51235e1e1f58 100644 --- a/doc/source/whatsnew/v0.24.0.rst +++ b/doc/source/whatsnew/v0.24.0.rst @@ -1440,7 +1440,7 @@ Numeric - :meth:`Series.agg` can now handle numpy NaN-aware methods like :func:`numpy.nansum` (:issue:`19629`) - Bug in :meth:`Series.rank` and :meth:`DataFrame.rank` when ``pct=True`` and more than 2:sup:`24` rows are present resulted in percentages greater than 1.0 (:issue:`18271`) - Calls such as :meth:`DataFrame.round` with a non-unique :meth:`CategoricalIndex` now return expected data. Previously, data would be improperly duplicated (:issue:`21809`). -- Added ``log10`` to the list of supported functions in :meth:`DataFrame.eval` (:issue:`24139`) +- Added ``log10``, `floor` and `ceil` to the list of supported functions in :meth:`DataFrame.eval` (:issue:`24139`, :issue:`24353`) - Logical operations ``&, |, ^`` between :class:`Series` and :class:`Index` will no longer raise ``ValueError`` (:issue:`22092`) - Checking PEP 3141 numbers in :func:`~pandas.api.types.is_scalar` function returns ``True`` (:issue:`22903`) - Reduction methods like :meth:`Series.sum` now accept the default value of ``keepdims=False`` when called from a NumPy ufunc, rather than raising a ``TypeError``. Full support for ``keepdims`` has not been implemented (:issue:`24356`). From f5b7c0e26c4b12a56fd61ca47cd500b0468fe61e Mon Sep 17 00:00:00 2001 From: Anjana S Date: Sun, 23 Dec 2018 17:59:51 +0530 Subject: [PATCH 03/17] raise error when function not supported by older version of numpy --- pandas/core/computation/ops.py | 2 +- pandas/tests/computation/test_eval.py | 17 ++++++++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/pandas/core/computation/ops.py b/pandas/core/computation/ops.py index c03bad1c0faa9..f2baa6d3777b5 100644 --- a/pandas/core/computation/ops.py +++ b/pandas/core/computation/ops.py @@ -542,7 +542,7 @@ def __unicode__(self): class FuncNode(object): def __init__(self, name): - if name not in _mathops: + if name not in _mathops or not hasattr(np, name) : raise ValueError( "\"{0}\" is not a supported function".format(name)) self.name = name diff --git a/pandas/tests/computation/test_eval.py b/pandas/tests/computation/test_eval.py index 52945edb14e58..08820d5a290c4 100644 --- a/pandas/tests/computation/test_eval.py +++ b/pandas/tests/computation/test_eval.py @@ -1627,10 +1627,21 @@ def test_unary_functions(self): a = df.a for fn in self.unary_fns: expr = "{0}(a)".format(fn) - got = self.eval(expr) with np.errstate(all='ignore'): - expect = getattr(np, fn)(a) - tm.assert_series_equal(got, expect, check_names=False) + if hasattr(np, fn): + got = self.eval(expr) + expect = getattr(np, fn)(a) + tm.assert_series_equal(got, expect, check_names=False) + + def test_all_unary_functions_not_supported_in_numpy_112(self): + df = DataFrame({'a': np.random.randn(10)}) + for fn in self.unary_fns: + expr = "{0}(a)".format(fn) + with np.errstate(all='ignore'): + if not hasattr(np, fn): + msg = f"\"{fn}\" is not a supported function" + with pytest.raises(ValueError, match=msg): + self.eval(expr) def test_binary_functions(self): df = DataFrame({'a': np.random.randn(10), From eb6c6217a7ee1fbb4074eb522bcbc3f1edcf4b22 Mon Sep 17 00:00:00 2001 From: Anjana S Date: Mon, 24 Dec 2018 22:06:37 +0530 Subject: [PATCH 04/17] Fix PEP8 issue --- pandas/core/computation/ops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/computation/ops.py b/pandas/core/computation/ops.py index f2baa6d3777b5..c5486e9c52b50 100644 --- a/pandas/core/computation/ops.py +++ b/pandas/core/computation/ops.py @@ -542,7 +542,7 @@ def __unicode__(self): class FuncNode(object): def __init__(self, name): - if name not in _mathops or not hasattr(np, name) : + if name not in _mathops or not hasattr(np, name): raise ValueError( "\"{0}\" is not a supported function".format(name)) self.name = name From 7e8a643caff329b10a7214745c1b0c2ce66cb2a9 Mon Sep 17 00:00:00 2001 From: Anjana S Date: Mon, 24 Dec 2018 22:13:42 +0530 Subject: [PATCH 05/17] Fix compilation issue in py 2.7 --- pandas/tests/computation/test_eval.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/computation/test_eval.py b/pandas/tests/computation/test_eval.py index 08820d5a290c4..64c08a9251f81 100644 --- a/pandas/tests/computation/test_eval.py +++ b/pandas/tests/computation/test_eval.py @@ -1639,7 +1639,7 @@ def test_all_unary_functions_not_supported_in_numpy_112(self): expr = "{0}(a)".format(fn) with np.errstate(all='ignore'): if not hasattr(np, fn): - msg = f"\"{fn}\" is not a supported function" + msg = '"{0}" is not a supported function'.format(fn) with pytest.raises(ValueError, match=msg): self.eval(expr) From 693449d67aacc14cdaf056acd780c9d31d2697af Mon Sep 17 00:00:00 2001 From: Anjana S Date: Mon, 24 Dec 2018 22:15:11 +0530 Subject: [PATCH 06/17] fix linting issue --- pandas/tests/computation/test_eval.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pandas/tests/computation/test_eval.py b/pandas/tests/computation/test_eval.py index 64c08a9251f81..80df70808bb5e 100644 --- a/pandas/tests/computation/test_eval.py +++ b/pandas/tests/computation/test_eval.py @@ -1635,6 +1635,7 @@ def test_unary_functions(self): def test_all_unary_functions_not_supported_in_numpy_112(self): df = DataFrame({'a': np.random.randn(10)}) + a = df.a for fn in self.unary_fns: expr = "{0}(a)".format(fn) with np.errstate(all='ignore'): From 13568a6664fcaf21e87d6c9a076be5159f4a1200 Mon Sep 17 00:00:00 2001 From: Anjana S Date: Tue, 25 Dec 2018 00:22:56 +0530 Subject: [PATCH 07/17] numexpr less than 2.6.9 --- pandas/core/computation/check.py | 2 ++ pandas/core/computation/expressions.py | 3 +- pandas/core/computation/ops.py | 8 ++++- pandas/tests/computation/test_eval.py | 45 +++++++++++++++++++------- 4 files changed, 45 insertions(+), 13 deletions(-) diff --git a/pandas/core/computation/check.py b/pandas/core/computation/check.py index d2d5e018063ff..d17bcdc969bc6 100644 --- a/pandas/core/computation/check.py +++ b/pandas/core/computation/check.py @@ -3,11 +3,13 @@ _NUMEXPR_INSTALLED = False _MIN_NUMEXPR_VERSION = "2.6.1" +_ne_version_under_2_6_9 = None try: import numexpr as ne ver = LooseVersion(ne.__version__) _NUMEXPR_INSTALLED = ver >= LooseVersion(_MIN_NUMEXPR_VERSION) + _ne_version_under_2_6_9 = ver < LooseVersion('2.6.9') if not _NUMEXPR_INSTALLED: warnings.warn( diff --git a/pandas/core/computation/expressions.py b/pandas/core/computation/expressions.py index d44fae624a91c..684185ed21d0f 100644 --- a/pandas/core/computation/expressions.py +++ b/pandas/core/computation/expressions.py @@ -11,7 +11,7 @@ import numpy as np import pandas.core.common as com -from pandas.core.computation.check import _NUMEXPR_INSTALLED +from pandas.core.computation.check import _NUMEXPR_INSTALLED, _ne_version_under_2_6_9 from pandas.core.config import get_option if _NUMEXPR_INSTALLED: @@ -20,6 +20,7 @@ _TEST_MODE = None _TEST_RESULT = None _USE_NUMEXPR = _NUMEXPR_INSTALLED +_ne_version_under2_6_9 = _ne_version_under_2_6_9 _evaluate = None _where = None diff --git a/pandas/core/computation/ops.py b/pandas/core/computation/ops.py index c5486e9c52b50..d1e652004d574 100644 --- a/pandas/core/computation/ops.py +++ b/pandas/core/computation/ops.py @@ -15,6 +15,8 @@ from pandas.core.base import StringMixin import pandas.core.common as com from pandas.core.computation.common import _ensure_decoded, _result_type_many +from pandas.core.computation.expressions import (_NUMEXPR_INSTALLED, + _ne_version_under_2_6_9) from pandas.core.computation.scope import _DEFAULT_GLOBALS from pandas.io.formats.printing import pprint_thing, pprint_thing_encoded @@ -26,6 +28,7 @@ 'arctan', 'arccosh', 'arcsinh', 'arctanh', 'abs', 'log10', 'floor', 'ceil') _binary_math_ops = ('arctan2',) +_ops_for_ne_gt_2_6_9 = ('floor', 'ceil') _mathops = _unary_math_ops + _binary_math_ops @@ -542,9 +545,12 @@ def __unicode__(self): class FuncNode(object): def __init__(self, name): - if name not in _mathops or not hasattr(np, name): + if name not in _mathops or ( + _NUMEXPR_INSTALLED and _ne_version_under_2_6_9 and name in _ops_for_ne_gt_2_6_9 + ): raise ValueError( "\"{0}\" is not a supported function".format(name)) + self.name = name self.func = getattr(np, name) diff --git a/pandas/tests/computation/test_eval.py b/pandas/tests/computation/test_eval.py index 80df70808bb5e..2fe4894cef479 100644 --- a/pandas/tests/computation/test_eval.py +++ b/pandas/tests/computation/test_eval.py @@ -31,6 +31,9 @@ assert_numpy_array_equal, assert_series_equal, assert_produces_warning) from pandas.compat import PY3, reduce +from pandas.core.computation.expressions import ( + _ne_version_under_2_6_9) + _series_frame_incompatible = _bool_ops_syms _scalar_skip = 'in', 'not in' @@ -54,6 +57,19 @@ def parser(request): return request.param +@pytest.fixture +def ne_lt_2_6_9(): + if not _ne_version_under_2_6_9: + pytest.skip("numexpr is > 2.6.9") + return 'numexpr' + +@pytest.fixture +def ne_gt_2_6_9(): + if _ne_version_under_2_6_9: + pytest.skip("numexpr is < 2.6.9") + return 'numexpr' + + def engine_has_neg_frac(engine): return _engines[engine].has_neg_frac @@ -1625,24 +1641,31 @@ def eval(self, *args, **kwargs): def test_unary_functions(self): df = DataFrame({'a': np.random.randn(10)}) a = df.a - for fn in self.unary_fns: + unary_functions = [x for x in self.unary_fns + if x not in ('floor', 'ceil')] + for fn in unary_functions: expr = "{0}(a)".format(fn) + got = self.eval(expr) with np.errstate(all='ignore'): - if hasattr(np, fn): - got = self.eval(expr) - expect = getattr(np, fn)(a) - tm.assert_series_equal(got, expect, check_names=False) + expect = getattr(np, fn)(a) + tm.assert_series_equal(got, expect, check_names=False) + + def test_floor_and_ceil_functions_raise_error(self, ne_lt_2_6_9): + for fn in ('floor', 'ceil'): + msg = "\"{0}\" is not a supported function".format(fn) + with pytest.raises(ValueError, match=msg): + expr = "{0}(100)".format(fn) + self.eval(expr) - def test_all_unary_functions_not_supported_in_numpy_112(self): + def test_floor_and_ceil_functions_evaluate_expressions(self, ne_gt_2_6_9): df = DataFrame({'a': np.random.randn(10)}) a = df.a - for fn in self.unary_fns: + for fn in ('floor', 'ceil'): expr = "{0}(a)".format(fn) + got = self.eval(expr) with np.errstate(all='ignore'): - if not hasattr(np, fn): - msg = '"{0}" is not a supported function'.format(fn) - with pytest.raises(ValueError, match=msg): - self.eval(expr) + expect = getattr(np, fn)(a) + tm.assert_series_equal(got, expect, check_names=False) def test_binary_functions(self): df = DataFrame({'a': np.random.randn(10), From f963077ba627a63dff3884091cbfcd95ba564afa Mon Sep 17 00:00:00 2001 From: Anjana S Date: Tue, 25 Dec 2018 01:27:18 +0530 Subject: [PATCH 08/17] refactored to pass _NUMEXPR_VERSION --- pandas/core/computation/check.py | 4 ++-- pandas/core/computation/expressions.py | 3 +-- pandas/core/computation/ops.py | 13 +++++++------ pandas/tests/computation/test_eval.py | 10 +++++----- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/pandas/core/computation/check.py b/pandas/core/computation/check.py index d17bcdc969bc6..762b811718493 100644 --- a/pandas/core/computation/check.py +++ b/pandas/core/computation/check.py @@ -3,13 +3,13 @@ _NUMEXPR_INSTALLED = False _MIN_NUMEXPR_VERSION = "2.6.1" -_ne_version_under_2_6_9 = None +_NUMEXPR_VERSION = None try: import numexpr as ne ver = LooseVersion(ne.__version__) _NUMEXPR_INSTALLED = ver >= LooseVersion(_MIN_NUMEXPR_VERSION) - _ne_version_under_2_6_9 = ver < LooseVersion('2.6.9') + _NUMEXPR_VERSION = ver if not _NUMEXPR_INSTALLED: warnings.warn( diff --git a/pandas/core/computation/expressions.py b/pandas/core/computation/expressions.py index 684185ed21d0f..d44fae624a91c 100644 --- a/pandas/core/computation/expressions.py +++ b/pandas/core/computation/expressions.py @@ -11,7 +11,7 @@ import numpy as np import pandas.core.common as com -from pandas.core.computation.check import _NUMEXPR_INSTALLED, _ne_version_under_2_6_9 +from pandas.core.computation.check import _NUMEXPR_INSTALLED from pandas.core.config import get_option if _NUMEXPR_INSTALLED: @@ -20,7 +20,6 @@ _TEST_MODE = None _TEST_RESULT = None _USE_NUMEXPR = _NUMEXPR_INSTALLED -_ne_version_under2_6_9 = _ne_version_under_2_6_9 _evaluate = None _where = None diff --git a/pandas/core/computation/ops.py b/pandas/core/computation/ops.py index d1e652004d574..1909b29ccf4b6 100644 --- a/pandas/core/computation/ops.py +++ b/pandas/core/computation/ops.py @@ -2,6 +2,7 @@ """ from datetime import datetime +from distutils.version import LooseVersion from functools import partial import operator as op @@ -14,6 +15,7 @@ import pandas as pd from pandas.core.base import StringMixin import pandas.core.common as com +from pandas.core.computation.check import _NUMEXPR_VERSION from pandas.core.computation.common import _ensure_decoded, _result_type_many from pandas.core.computation.expressions import (_NUMEXPR_INSTALLED, _ne_version_under_2_6_9) @@ -25,10 +27,11 @@ _unary_math_ops = ('sin', 'cos', 'exp', 'log', 'expm1', 'log1p', 'sqrt', 'sinh', 'cosh', 'tanh', 'arcsin', 'arccos', - 'arctan', 'arccosh', 'arcsinh', 'arctanh', 'abs', 'log10', - 'floor', 'ceil') + 'arctan', 'arccosh', 'arcsinh', 'arctanh', 'abs', 'log10') _binary_math_ops = ('arctan2',) -_ops_for_ne_gt_2_6_9 = ('floor', 'ceil') +if _NUMEXPR_INSTALLED and _NUMEXPR_VERSION >= LooseVersion('2.6.9'): + _unary_math_ops += ('floor', 'ceil') + _mathops = _unary_math_ops + _binary_math_ops @@ -545,9 +548,7 @@ def __unicode__(self): class FuncNode(object): def __init__(self, name): - if name not in _mathops or ( - _NUMEXPR_INSTALLED and _ne_version_under_2_6_9 and name in _ops_for_ne_gt_2_6_9 - ): + if name not in _mathops: raise ValueError( "\"{0}\" is not a supported function".format(name)) diff --git a/pandas/tests/computation/test_eval.py b/pandas/tests/computation/test_eval.py index 2fe4894cef479..4b5d36db340da 100644 --- a/pandas/tests/computation/test_eval.py +++ b/pandas/tests/computation/test_eval.py @@ -1,5 +1,6 @@ import warnings import operator +from distutils.version import LooseVersion from itertools import product import pytest @@ -31,8 +32,7 @@ assert_numpy_array_equal, assert_series_equal, assert_produces_warning) from pandas.compat import PY3, reduce -from pandas.core.computation.expressions import ( - _ne_version_under_2_6_9) +from pandas.core.computation.check import _NUMEXPR_VERSION _series_frame_incompatible = _bool_ops_syms @@ -59,13 +59,13 @@ def parser(request): @pytest.fixture def ne_lt_2_6_9(): - if not _ne_version_under_2_6_9: - pytest.skip("numexpr is > 2.6.9") + if _NUMEXPR_INSTALLED and _NUMEXPR_VERSION >= LooseVersion('2.6.9'): + pytest.skip("numexpr is >= 2.6.9") return 'numexpr' @pytest.fixture def ne_gt_2_6_9(): - if _ne_version_under_2_6_9: + if _NUMEXPR_INSTALLED and _NUMEXPR_VERSION < LooseVersion('2.6.9'): pytest.skip("numexpr is < 2.6.9") return 'numexpr' From c8a8a02ba42aabda9a87d15a043e77dbd9fbb839 Mon Sep 17 00:00:00 2001 From: Anjana S Date: Tue, 25 Dec 2018 02:06:22 +0530 Subject: [PATCH 09/17] using self.unary_fns removed unused imports --- pandas/core/computation/ops.py | 3 +-- pandas/tests/computation/test_eval.py | 15 ++------------- 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/pandas/core/computation/ops.py b/pandas/core/computation/ops.py index 1909b29ccf4b6..07dd66c179ed1 100644 --- a/pandas/core/computation/ops.py +++ b/pandas/core/computation/ops.py @@ -17,8 +17,7 @@ import pandas.core.common as com from pandas.core.computation.check import _NUMEXPR_VERSION from pandas.core.computation.common import _ensure_decoded, _result_type_many -from pandas.core.computation.expressions import (_NUMEXPR_INSTALLED, - _ne_version_under_2_6_9) +from pandas.core.computation.expressions import _NUMEXPR_INSTALLED from pandas.core.computation.scope import _DEFAULT_GLOBALS from pandas.io.formats.printing import pprint_thing, pprint_thing_encoded diff --git a/pandas/tests/computation/test_eval.py b/pandas/tests/computation/test_eval.py index 4b5d36db340da..3eeac293ea99c 100644 --- a/pandas/tests/computation/test_eval.py +++ b/pandas/tests/computation/test_eval.py @@ -1641,9 +1641,8 @@ def eval(self, *args, **kwargs): def test_unary_functions(self): df = DataFrame({'a': np.random.randn(10)}) a = df.a - unary_functions = [x for x in self.unary_fns - if x not in ('floor', 'ceil')] - for fn in unary_functions: + + for fn in self.unary_fns: expr = "{0}(a)".format(fn) got = self.eval(expr) with np.errstate(all='ignore'): @@ -1657,16 +1656,6 @@ def test_floor_and_ceil_functions_raise_error(self, ne_lt_2_6_9): expr = "{0}(100)".format(fn) self.eval(expr) - def test_floor_and_ceil_functions_evaluate_expressions(self, ne_gt_2_6_9): - df = DataFrame({'a': np.random.randn(10)}) - a = df.a - for fn in ('floor', 'ceil'): - expr = "{0}(a)".format(fn) - got = self.eval(expr) - with np.errstate(all='ignore'): - expect = getattr(np, fn)(a) - tm.assert_series_equal(got, expect, check_names=False) - def test_binary_functions(self): df = DataFrame({'a': np.random.randn(10), 'b': np.random.randn(10)}) From bdd9c6844c75c38a824eab8924eae50eadb919e6 Mon Sep 17 00:00:00 2001 From: Anjana S Date: Tue, 25 Dec 2018 14:56:37 +0530 Subject: [PATCH 10/17] fix linting issue --- pandas/tests/computation/test_eval.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pandas/tests/computation/test_eval.py b/pandas/tests/computation/test_eval.py index 3eeac293ea99c..7ef7fdf1ba77f 100644 --- a/pandas/tests/computation/test_eval.py +++ b/pandas/tests/computation/test_eval.py @@ -63,6 +63,7 @@ def ne_lt_2_6_9(): pytest.skip("numexpr is >= 2.6.9") return 'numexpr' + @pytest.fixture def ne_gt_2_6_9(): if _NUMEXPR_INSTALLED and _NUMEXPR_VERSION < LooseVersion('2.6.9'): From 1e6ea3d8a2997eac577a1047d450d1bfd869af4b Mon Sep 17 00:00:00 2001 From: Anjana S Date: Tue, 25 Dec 2018 17:15:55 +0530 Subject: [PATCH 11/17] restored sys modules after import --- pandas/core/computation/check.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pandas/core/computation/check.py b/pandas/core/computation/check.py index 762b811718493..d003b65809097 100644 --- a/pandas/core/computation/check.py +++ b/pandas/core/computation/check.py @@ -1,3 +1,4 @@ +import sys from distutils.version import LooseVersion import warnings @@ -6,10 +7,12 @@ _NUMEXPR_VERSION = None try: + modules = sys.modules.copy() import numexpr as ne ver = LooseVersion(ne.__version__) _NUMEXPR_INSTALLED = ver >= LooseVersion(_MIN_NUMEXPR_VERSION) _NUMEXPR_VERSION = ver + sys.modules = modules if not _NUMEXPR_INSTALLED: warnings.warn( @@ -21,4 +24,4 @@ except ImportError: # pragma: no cover pass -__all__ = ['_NUMEXPR_INSTALLED'] +__all__ = ['_NUMEXPR_INSTALLED', '_NUMEXPR_VERSION'] From a3a4b9e4acdf0596dfdd0c2164a0f068e588bef2 Mon Sep 17 00:00:00 2001 From: Anjana S Date: Tue, 25 Dec 2018 17:17:41 +0530 Subject: [PATCH 12/17] remove unused ne_gt_2_6_9 --- pandas/tests/computation/test_eval.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/pandas/tests/computation/test_eval.py b/pandas/tests/computation/test_eval.py index 7ef7fdf1ba77f..99125a1f04b0c 100644 --- a/pandas/tests/computation/test_eval.py +++ b/pandas/tests/computation/test_eval.py @@ -64,13 +64,6 @@ def ne_lt_2_6_9(): return 'numexpr' -@pytest.fixture -def ne_gt_2_6_9(): - if _NUMEXPR_INSTALLED and _NUMEXPR_VERSION < LooseVersion('2.6.9'): - pytest.skip("numexpr is < 2.6.9") - return 'numexpr' - - def engine_has_neg_frac(engine): return _engines[engine].has_neg_frac From bc154e25e281087835d8931419685fcfefa519da Mon Sep 17 00:00:00 2001 From: Anjana S Date: Tue, 25 Dec 2018 17:47:55 +0530 Subject: [PATCH 13/17] Revert "restored sys modules after import" This reverts commit 5933d5e794d0ff4661e1e7d341c98f7f0dd7b802. --- pandas/core/computation/check.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pandas/core/computation/check.py b/pandas/core/computation/check.py index d003b65809097..762b811718493 100644 --- a/pandas/core/computation/check.py +++ b/pandas/core/computation/check.py @@ -1,4 +1,3 @@ -import sys from distutils.version import LooseVersion import warnings @@ -7,12 +6,10 @@ _NUMEXPR_VERSION = None try: - modules = sys.modules.copy() import numexpr as ne ver = LooseVersion(ne.__version__) _NUMEXPR_INSTALLED = ver >= LooseVersion(_MIN_NUMEXPR_VERSION) _NUMEXPR_VERSION = ver - sys.modules = modules if not _NUMEXPR_INSTALLED: warnings.warn( @@ -24,4 +21,4 @@ except ImportError: # pragma: no cover pass -__all__ = ['_NUMEXPR_INSTALLED', '_NUMEXPR_VERSION'] +__all__ = ['_NUMEXPR_INSTALLED'] From caaf634ed03aa227fbd988f50be80efea7d53d7b Mon Sep 17 00:00:00 2001 From: Anjana S Date: Tue, 25 Dec 2018 17:58:45 +0530 Subject: [PATCH 14/17] exporting _NUMEXPR_VERSION as well --- pandas/core/computation/check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/computation/check.py b/pandas/core/computation/check.py index 762b811718493..da89bde56fe18 100644 --- a/pandas/core/computation/check.py +++ b/pandas/core/computation/check.py @@ -21,4 +21,4 @@ except ImportError: # pragma: no cover pass -__all__ = ['_NUMEXPR_INSTALLED'] +__all__ = ['_NUMEXPR_INSTALLED', '_NUMEXPR_VERSION'] From bb07d26b0937f208766e728186138e239aafde73 Mon Sep 17 00:00:00 2001 From: Anjana S Date: Tue, 25 Dec 2018 22:46:29 +0530 Subject: [PATCH 15/17] fix imports --- pandas/core/computation/expressions.py | 2 +- pandas/core/computation/ops.py | 3 +-- pandas/tests/computation/test_eval.py | 3 +-- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/pandas/core/computation/expressions.py b/pandas/core/computation/expressions.py index d44fae624a91c..3c67161cd9fea 100644 --- a/pandas/core/computation/expressions.py +++ b/pandas/core/computation/expressions.py @@ -11,7 +11,7 @@ import numpy as np import pandas.core.common as com -from pandas.core.computation.check import _NUMEXPR_INSTALLED +from pandas.core.computation.check import _NUMEXPR_INSTALLED, _NUMEXPR_VERSION from pandas.core.config import get_option if _NUMEXPR_INSTALLED: diff --git a/pandas/core/computation/ops.py b/pandas/core/computation/ops.py index 07dd66c179ed1..604a5d61ee45a 100644 --- a/pandas/core/computation/ops.py +++ b/pandas/core/computation/ops.py @@ -15,9 +15,8 @@ import pandas as pd from pandas.core.base import StringMixin import pandas.core.common as com -from pandas.core.computation.check import _NUMEXPR_VERSION +from pandas.core.computation.check import _NUMEXPR_INSTALLED, _NUMEXPR_VERSION from pandas.core.computation.common import _ensure_decoded, _result_type_many -from pandas.core.computation.expressions import _NUMEXPR_INSTALLED from pandas.core.computation.scope import _DEFAULT_GLOBALS from pandas.io.formats.printing import pprint_thing, pprint_thing_encoded diff --git a/pandas/tests/computation/test_eval.py b/pandas/tests/computation/test_eval.py index 99125a1f04b0c..b8fa7b0247008 100644 --- a/pandas/tests/computation/test_eval.py +++ b/pandas/tests/computation/test_eval.py @@ -18,7 +18,7 @@ from pandas.core.computation.engines import _engines, NumExprClobberingError from pandas.core.computation.expr import PythonExprVisitor, PandasExprVisitor from pandas.core.computation.expressions import ( - _USE_NUMEXPR, _NUMEXPR_INSTALLED) + _USE_NUMEXPR, _NUMEXPR_INSTALLED, _NUMEXPR_VERSION) from pandas.core.computation.ops import ( _binary_ops_dict, _special_case_arith_ops_syms, @@ -32,7 +32,6 @@ assert_numpy_array_equal, assert_series_equal, assert_produces_warning) from pandas.compat import PY3, reduce -from pandas.core.computation.check import _NUMEXPR_VERSION _series_frame_incompatible = _bool_ops_syms From 38115f6bc9bec24b7535f7459ff9eeb6be01f5e6 Mon Sep 17 00:00:00 2001 From: Anjana S Date: Tue, 25 Dec 2018 23:36:50 +0530 Subject: [PATCH 16/17] fixed imports --- pandas/core/computation/expressions.py | 2 +- pandas/tests/computation/test_eval.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pandas/core/computation/expressions.py b/pandas/core/computation/expressions.py index 3c67161cd9fea..d44fae624a91c 100644 --- a/pandas/core/computation/expressions.py +++ b/pandas/core/computation/expressions.py @@ -11,7 +11,7 @@ import numpy as np import pandas.core.common as com -from pandas.core.computation.check import _NUMEXPR_INSTALLED, _NUMEXPR_VERSION +from pandas.core.computation.check import _NUMEXPR_INSTALLED from pandas.core.config import get_option if _NUMEXPR_INSTALLED: diff --git a/pandas/tests/computation/test_eval.py b/pandas/tests/computation/test_eval.py index b8fa7b0247008..dec30c202f17b 100644 --- a/pandas/tests/computation/test_eval.py +++ b/pandas/tests/computation/test_eval.py @@ -15,10 +15,11 @@ from pandas.util.testing import makeCustomDataframe as mkdf from pandas.core.computation import pytables +from pandas.core.computation.check import _NUMEXPR_VERSION from pandas.core.computation.engines import _engines, NumExprClobberingError from pandas.core.computation.expr import PythonExprVisitor, PandasExprVisitor from pandas.core.computation.expressions import ( - _USE_NUMEXPR, _NUMEXPR_INSTALLED, _NUMEXPR_VERSION) + _USE_NUMEXPR, _NUMEXPR_INSTALLED) from pandas.core.computation.ops import ( _binary_ops_dict, _special_case_arith_ops_syms, From 3ccb40c8608b4a41b2b16b566f94aeb31f66608f Mon Sep 17 00:00:00 2001 From: Anjana S Date: Sun, 30 Dec 2018 10:57:04 +0530 Subject: [PATCH 17/17] Fix numexpr version issue --- pandas/core/computation/ops.py | 16 ++++++++++------ pandas/tests/computation/test_eval.py | 20 +++++++++++++++++--- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/pandas/core/computation/ops.py b/pandas/core/computation/ops.py index 604a5d61ee45a..8c3218a976b6b 100644 --- a/pandas/core/computation/ops.py +++ b/pandas/core/computation/ops.py @@ -15,7 +15,6 @@ import pandas as pd from pandas.core.base import StringMixin import pandas.core.common as com -from pandas.core.computation.check import _NUMEXPR_INSTALLED, _NUMEXPR_VERSION from pandas.core.computation.common import _ensure_decoded, _result_type_many from pandas.core.computation.scope import _DEFAULT_GLOBALS @@ -25,10 +24,10 @@ _unary_math_ops = ('sin', 'cos', 'exp', 'log', 'expm1', 'log1p', 'sqrt', 'sinh', 'cosh', 'tanh', 'arcsin', 'arccos', - 'arctan', 'arccosh', 'arcsinh', 'arctanh', 'abs', 'log10') + 'arctan', 'arccosh', 'arcsinh', 'arctanh', 'abs', 'log10', + 'floor', 'ceil' + ) _binary_math_ops = ('arctan2',) -if _NUMEXPR_INSTALLED and _NUMEXPR_VERSION >= LooseVersion('2.6.9'): - _unary_math_ops += ('floor', 'ceil') _mathops = _unary_math_ops + _binary_math_ops @@ -544,9 +543,14 @@ def __unicode__(self): class FuncNode(object): - def __init__(self, name): - if name not in _mathops: + from pandas.core.computation.check import (_NUMEXPR_INSTALLED, + _NUMEXPR_VERSION) + if name not in _mathops or ( + _NUMEXPR_INSTALLED and + _NUMEXPR_VERSION < LooseVersion('2.6.9') and + name in ('floor', 'ceil') + ): raise ValueError( "\"{0}\" is not a supported function".format(name)) diff --git a/pandas/tests/computation/test_eval.py b/pandas/tests/computation/test_eval.py index dec30c202f17b..1649c99384ef2 100644 --- a/pandas/tests/computation/test_eval.py +++ b/pandas/tests/computation/test_eval.py @@ -64,6 +64,18 @@ def ne_lt_2_6_9(): return 'numexpr' +@pytest.fixture +def unary_fns_for_ne(): + if _NUMEXPR_INSTALLED: + if _NUMEXPR_VERSION >= LooseVersion('2.6.9'): + return _unary_math_ops + else: + return tuple(x for x in _unary_math_ops + if x not in ("floor", "ceil")) + else: + pytest.skip("numexpr is not present") + + def engine_has_neg_frac(engine): return _engines[engine].has_neg_frac @@ -1632,18 +1644,20 @@ def eval(self, *args, **kwargs): kwargs['level'] = kwargs.pop('level', 0) + 1 return pd.eval(*args, **kwargs) - def test_unary_functions(self): + def test_unary_functions(self, unary_fns_for_ne): df = DataFrame({'a': np.random.randn(10)}) a = df.a - for fn in self.unary_fns: + for fn in unary_fns_for_ne: expr = "{0}(a)".format(fn) got = self.eval(expr) with np.errstate(all='ignore'): expect = getattr(np, fn)(a) tm.assert_series_equal(got, expect, check_names=False) - def test_floor_and_ceil_functions_raise_error(self, ne_lt_2_6_9): + def test_floor_and_ceil_functions_raise_error(self, + ne_lt_2_6_9, + unary_fns_for_ne): for fn in ('floor', 'ceil'): msg = "\"{0}\" is not a supported function".format(fn) with pytest.raises(ValueError, match=msg):