Skip to content

Commit 4e8b6c7

Browse files
committed
NumExpr decorator
1 parent 0912329 commit 4e8b6c7

File tree

4 files changed

+25
-89
lines changed

4 files changed

+25
-89
lines changed

pandas/tests/computation/test_eval.py

+7-10
Original file line numberDiff line numberDiff line change
@@ -97,11 +97,11 @@ def _is_py3_complex_incompat(result, expected):
9797
_good_arith_ops = com.difference(_arith_ops_syms, _special_case_arith_ops_syms)
9898

9999

100+
@td.skip_if_no_ne
100101
class TestEvalNumexprPandas(object):
101102

102103
@classmethod
103104
def setup_class(cls):
104-
tm.skip_if_no_ne()
105105
import numexpr as ne
106106
cls.ne = ne
107107
cls.engine = 'numexpr'
@@ -374,7 +374,6 @@ def check_single_invert_op(self, lhs, cmp1, rhs):
374374
tm.assert_almost_equal(expected, result)
375375

376376
for engine in self.current_engines:
377-
tm.skip_if_no_ne(engine)
378377
tm.assert_almost_equal(result, pd.eval('~elb', engine=engine,
379378
parser=self.parser))
380379

@@ -400,7 +399,6 @@ def check_compound_invert_op(self, lhs, cmp1, rhs):
400399

401400
# make sure the other engines work the same as this one
402401
for engine in self.current_engines:
403-
tm.skip_if_no_ne(engine)
404402
ev = pd.eval(ex, engine=self.engine, parser=self.parser)
405403
tm.assert_almost_equal(ev, result)
406404

@@ -731,12 +729,12 @@ def test_disallow_python_keywords(self):
731729
df.query('lambda == 0')
732730

733731

732+
@td.skip_if_no_ne
734733
class TestEvalNumexprPython(TestEvalNumexprPandas):
735734

736735
@classmethod
737736
def setup_class(cls):
738737
super(TestEvalNumexprPython, cls).setup_class()
739-
tm.skip_if_no_ne()
740738
import numexpr as ne
741739
cls.ne = ne
742740
cls.engine = 'numexpr'
@@ -1078,11 +1076,11 @@ def test_performance_warning_for_poor_alignment(self, engine, parser):
10781076
# ------------------------------------
10791077
# Slightly more complex ops
10801078

1079+
@td.skip_if_no_ne
10811080
class TestOperationsNumExprPandas(object):
10821081

10831082
@classmethod
10841083
def setup_class(cls):
1085-
tm.skip_if_no_ne()
10861084
cls.engine = 'numexpr'
10871085
cls.parser = 'pandas'
10881086
cls.arith_ops = expr._arith_ops_syms + expr._cmp_ops_syms
@@ -1528,14 +1526,14 @@ def test_simple_in_ops(self):
15281526
parser=self.parser)
15291527

15301528

1529+
@td.skip_if_no_ne
15311530
class TestOperationsNumExprPython(TestOperationsNumExprPandas):
15321531

15331532
@classmethod
15341533
def setup_class(cls):
15351534
super(TestOperationsNumExprPython, cls).setup_class()
15361535
cls.engine = 'numexpr'
15371536
cls.parser = 'python'
1538-
tm.skip_if_no_ne(cls.engine)
15391537
cls.arith_ops = expr._arith_ops_syms + expr._cmp_ops_syms
15401538
cls.arith_ops = filter(lambda x: x not in ('in', 'not in'),
15411539
cls.arith_ops)
@@ -1623,11 +1621,11 @@ def setup_class(cls):
16231621
cls.arith_ops = expr._arith_ops_syms + expr._cmp_ops_syms
16241622

16251623

1624+
@td.skip_if_no_ne
16261625
class TestMathPythonPython(object):
16271626

16281627
@classmethod
16291628
def setup_class(cls):
1630-
tm.skip_if_no_ne()
16311629
cls.engine = 'python'
16321630
cls.parser = 'pandas'
16331631
cls.unary_fns = _unary_math_ops
@@ -1782,15 +1780,15 @@ def test_no_new_globals(self, engine, parser):
17821780
assert gbls == gbls2
17831781

17841782

1783+
@td.skip_if_no_ne
17851784
def test_invalid_engine():
1786-
tm.skip_if_no_ne()
17871785
tm.assert_raises_regex(KeyError, 'Invalid engine \'asdf\' passed',
17881786
pd.eval, 'x + y', local_dict={'x': 1, 'y': 2},
17891787
engine='asdf')
17901788

17911789

1790+
@td.skip_if_no_ne
17921791
def test_invalid_parser():
1793-
tm.skip_if_no_ne()
17941792
tm.assert_raises_regex(KeyError, 'Invalid parser \'asdf\' passed',
17951793
pd.eval, 'x + y', local_dict={'x': 1, 'y': 2},
17961794
parser='asdf')
@@ -1803,7 +1801,6 @@ def test_invalid_parser():
18031801
@pytest.mark.parametrize('engine', _parsers)
18041802
@pytest.mark.parametrize('parser', _parsers)
18051803
def test_disallowed_nodes(engine, parser):
1806-
tm.skip_if_no_ne(engine)
18071804
VisitorClass = _parsers[parser]
18081805
uns_ops = VisitorClass.unsupported_nodes
18091806
inst = VisitorClass('x + 1', engine, parser)

pandas/tests/frame/test_query_eval.py

+11-65
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,14 @@
1717
makeCustomDataframe as mkdf)
1818

1919
import pandas.util.testing as tm
20+
import pandas.util._test_decorators as td
2021
from pandas.core.computation.check import _NUMEXPR_INSTALLED
2122

2223
from pandas.tests.frame.common import TestData
2324

2425

2526
PARSERS = 'python', 'pandas'
26-
ENGINES = 'python', 'numexpr'
27+
ENGINES = 'python', pytest.param('numexpr', marks=td.skip_if_no_ne)
2728

2829

2930
@pytest.fixture(params=PARSERS, ids=lambda x: x)
@@ -41,13 +42,6 @@ def skip_if_no_pandas_parser(parser):
4142
pytest.skip("cannot evaluate with parser {0!r}".format(parser))
4243

4344

44-
def skip_if_no_ne(engine='numexpr'):
45-
if engine == 'numexpr':
46-
if not _NUMEXPR_INSTALLED:
47-
pytest.skip("cannot query engine numexpr when numexpr not "
48-
"installed")
49-
50-
5145
class TestCompat(object):
5246

5347
def setup_method(self, method):
@@ -175,7 +169,6 @@ def test_eval_resolvers_as_list(self):
175169
class TestDataFrameQueryWithMultiIndex(object):
176170

177171
def test_query_with_named_multiindex(self, parser, engine):
178-
tm.skip_if_no_ne(engine)
179172
skip_if_no_pandas_parser(parser)
180173
a = np.random.choice(['red', 'green'], size=10)
181174
b = np.random.choice(['eggs', 'ham'], size=10)
@@ -225,7 +218,6 @@ def test_query_with_named_multiindex(self, parser, engine):
225218
assert_frame_equal(res2, exp)
226219

227220
def test_query_with_unnamed_multiindex(self, parser, engine):
228-
tm.skip_if_no_ne(engine)
229221
skip_if_no_pandas_parser(parser)
230222
a = np.random.choice(['red', 'green'], size=10)
231223
b = np.random.choice(['eggs', 'ham'], size=10)
@@ -316,7 +308,6 @@ def test_query_with_unnamed_multiindex(self, parser, engine):
316308
assert_frame_equal(res2, exp)
317309

318310
def test_query_with_partially_named_multiindex(self, parser, engine):
319-
tm.skip_if_no_ne(engine)
320311
skip_if_no_pandas_parser(parser)
321312
a = np.random.choice(['red', 'green'], size=10)
322313
b = np.arange(10)
@@ -370,27 +361,25 @@ def to_series(mi, level):
370361
raise AssertionError("object must be a Series or Index")
371362

372363
def test_raise_on_panel_with_multiindex(self, parser, engine):
373-
tm.skip_if_no_ne()
374364
p = tm.makePanel(7)
375365
p.items = tm.makeCustomIndex(len(p.items), nlevels=2)
376366
with pytest.raises(NotImplementedError):
377367
pd.eval('p + 1', parser=parser, engine=engine)
378368

379369
def test_raise_on_panel4d_with_multiindex(self, parser, engine):
380-
tm.skip_if_no_ne()
381370
p4d = tm.makePanel4D(7)
382371
p4d.items = tm.makeCustomIndex(len(p4d.items), nlevels=2)
383372
with pytest.raises(NotImplementedError):
384373
pd.eval('p4d + 1', parser=parser, engine=engine)
385374

386375

376+
@td.skip_if_no_ne
387377
class TestDataFrameQueryNumExprPandas(object):
388378

389379
@classmethod
390380
def setup_class(cls):
391381
cls.engine = 'numexpr'
392382
cls.parser = 'pandas'
393-
tm.skip_if_no_ne(cls.engine)
394383

395384
@classmethod
396385
def teardown_class(cls):
@@ -714,14 +703,14 @@ def test_inf(self):
714703
assert_frame_equal(result, expected)
715704

716705

706+
@td.skip_if_no_ne
717707
class TestDataFrameQueryNumExprPython(TestDataFrameQueryNumExprPandas):
718708

719709
@classmethod
720710
def setup_class(cls):
721711
super(TestDataFrameQueryNumExprPython, cls).setup_class()
722712
cls.engine = 'numexpr'
723713
cls.parser = 'python'
724-
tm.skip_if_no_ne(cls.engine)
725714
cls.frame = TestData().frame
726715

727716
def test_date_query_no_attribute_access(self):
@@ -859,7 +848,6 @@ def test_query_builtin(self):
859848
class TestDataFrameQueryStrings(object):
860849

861850
def test_str_query_method(self, parser, engine):
862-
tm.skip_if_no_ne(engine)
863851
df = DataFrame(randn(10, 1), columns=['b'])
864852
df['strings'] = Series(list('aabbccddee'))
865853
expect = df[df.strings == 'a']
@@ -896,7 +884,6 @@ def test_str_query_method(self, parser, engine):
896884
assert_frame_equal(res, df[~df.strings.isin(['a'])])
897885

898886
def test_str_list_query_method(self, parser, engine):
899-
tm.skip_if_no_ne(engine)
900887
df = DataFrame(randn(10, 1), columns=['b'])
901888
df['strings'] = Series(list('aabbccddee'))
902889
expect = df[df.strings.isin(['a', 'b'])]
@@ -935,7 +922,6 @@ def test_str_list_query_method(self, parser, engine):
935922
assert_frame_equal(res, expect)
936923

937924
def test_query_with_string_columns(self, parser, engine):
938-
tm.skip_if_no_ne(engine)
939925
df = DataFrame({'a': list('aaaabbbbcccc'),
940926
'b': list('aabbccddeeff'),
941927
'c': np.random.randint(5, size=12),
@@ -956,7 +942,6 @@ def test_query_with_string_columns(self, parser, engine):
956942
df.query('a in b and c < d', parser=parser, engine=engine)
957943

958944
def test_object_array_eq_ne(self, parser, engine):
959-
tm.skip_if_no_ne(engine)
960945
df = DataFrame({'a': list('aaaabbbbcccc'),
961946
'b': list('aabbccddeeff'),
962947
'c': np.random.randint(5, size=12),
@@ -970,7 +955,6 @@ def test_object_array_eq_ne(self, parser, engine):
970955
assert_frame_equal(res, exp)
971956

972957
def test_query_with_nested_strings(self, parser, engine):
973-
tm.skip_if_no_ne(engine)
974958
skip_if_no_pandas_parser(parser)
975959
raw = """id event timestamp
976960
1 "page 1 load" 1/1/2014 0:00:01
@@ -995,15 +979,13 @@ def test_query_with_nested_strings(self, parser, engine):
995979

996980
def test_query_with_nested_special_character(self, parser, engine):
997981
skip_if_no_pandas_parser(parser)
998-
tm.skip_if_no_ne(engine)
999982
df = DataFrame({'a': ['a', 'b', 'test & test'],
1000983
'b': [1, 2, 3]})
1001984
res = df.query('a == "test & test"', parser=parser, engine=engine)
1002985
expec = df[df.a == 'test & test']
1003986
assert_frame_equal(res, expec)
1004987

1005988
def test_query_lex_compare_strings(self, parser, engine):
1006-
tm.skip_if_no_ne(engine=engine)
1007989
import operator as opr
1008990

1009991
a = Series(np.random.choice(list('abcde'), 20))
@@ -1018,7 +1000,6 @@ def test_query_lex_compare_strings(self, parser, engine):
10181000
assert_frame_equal(res, expected)
10191001

10201002
def test_query_single_element_booleans(self, parser, engine):
1021-
tm.skip_if_no_ne(engine)
10221003
columns = 'bid', 'bidsize', 'ask', 'asksize'
10231004
data = np.random.randint(2, size=(1, len(columns))).astype(bool)
10241005
df = DataFrame(data, columns=columns)
@@ -1027,7 +1008,6 @@ def test_query_single_element_booleans(self, parser, engine):
10271008
assert_frame_equal(res, expected)
10281009

10291010
def test_query_string_scalar_variable(self, parser, engine):
1030-
tm.skip_if_no_ne(engine)
10311011
skip_if_no_pandas_parser(parser)
10321012
df = pd.DataFrame({'Symbol': ['BUD US', 'BUD US', 'IBM US', 'IBM US'],
10331013
'Price': [109.70, 109.72, 183.30, 183.35]})
@@ -1037,63 +1017,29 @@ def test_query_string_scalar_variable(self, parser, engine):
10371017
assert_frame_equal(e, r)
10381018

10391019

1040-
class TestDataFrameEvalNumExprPandas(object):
1041-
1042-
@classmethod
1043-
def setup_class(cls):
1044-
cls.engine = 'numexpr'
1045-
cls.parser = 'pandas'
1046-
tm.skip_if_no_ne()
1020+
class TestDataFrameEvalWithFrame(object):
10471021

10481022
def setup_method(self, method):
10491023
self.frame = DataFrame(randn(10, 3), columns=list('abc'))
10501024

10511025
def teardown_method(self, method):
10521026
del self.frame
10531027

1054-
def test_simple_expr(self):
1055-
res = self.frame.eval('a + b', engine=self.engine, parser=self.parser)
1028+
def test_simple_expr(self, parser, engine):
1029+
res = self.frame.eval('a + b', engine=engine, parser=parser)
10561030
expect = self.frame.a + self.frame.b
10571031
assert_series_equal(res, expect)
10581032

1059-
def test_bool_arith_expr(self):
1060-
res = self.frame.eval('a[a < 1] + b', engine=self.engine,
1061-
parser=self.parser)
1033+
def test_bool_arith_expr(self, parser, engine):
1034+
res = self.frame.eval('a[a < 1] + b', engine=engine, parser=parser)
10621035
expect = self.frame.a[self.frame.a < 1] + self.frame.b
10631036
assert_series_equal(res, expect)
10641037

1065-
def test_invalid_type_for_operator_raises(self):
1038+
def test_invalid_type_for_operator_raises(self, parser, engine):
10661039
df = DataFrame({'a': [1, 2], 'b': ['c', 'd']})
10671040
ops = '+', '-', '*', '/'
10681041
for op in ops:
10691042
with tm.assert_raises_regex(TypeError,
10701043
"unsupported operand type\(s\) "
10711044
"for .+: '.+' and '.+'"):
1072-
df.eval('a {0} b'.format(op), engine=self.engine,
1073-
parser=self.parser)
1074-
1075-
1076-
class TestDataFrameEvalNumExprPython(TestDataFrameEvalNumExprPandas):
1077-
1078-
@classmethod
1079-
def setup_class(cls):
1080-
super(TestDataFrameEvalNumExprPython, cls).setup_class()
1081-
cls.engine = 'numexpr'
1082-
cls.parser = 'python'
1083-
tm.skip_if_no_ne(cls.engine)
1084-
1085-
1086-
class TestDataFrameEvalPythonPandas(TestDataFrameEvalNumExprPandas):
1087-
1088-
@classmethod
1089-
def setup_class(cls):
1090-
super(TestDataFrameEvalPythonPandas, cls).setup_class()
1091-
cls.engine = 'python'
1092-
cls.parser = 'pandas'
1093-
1094-
1095-
class TestDataFrameEvalPythonPython(TestDataFrameEvalNumExprPython):
1096-
1097-
@classmethod
1098-
def setup_class(cls):
1099-
cls.engine = cls.parser = 'python'
1045+
df.eval('a {0} b'.format(op), engine=engine, parser=parser)

pandas/util/_test_decorators.py

+7
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ def test_foo():
3030

3131
from pandas.compat import (is_platform_windows, is_platform_32bit, PY3,
3232
import_lzma)
33+
from pandas.core.computation.expressions import (_USE_NUMEXPR,
34+
_NUMEXPR_INSTALLED)
3335

3436

3537
def safe_import(mod_name, min_version=None):
@@ -163,3 +165,8 @@ def decorated_func(func):
163165
reason="Missing SciPy requirement")
164166
skip_if_no_lzma = pytest.mark.skipif(_skip_if_no_lzma(),
165167
reason="need backports.lzma to run")
168+
skip_if_no_ne = pytest.mark.skipif(not _USE_NUMEXPR,
169+
reason="numexpr enabled->{enabled}, "
170+
"installed->{installed}".format(
171+
enabled=_USE_NUMEXPR,
172+
installed=_NUMEXPR_INSTALLED))

pandas/util/testing.py

-14
Original file line numberDiff line numberDiff line change
@@ -336,20 +336,6 @@ def _skip_if_no_scipy():
336336
pytest.importorskip("scipy.interpolate")
337337

338338

339-
def skip_if_no_ne(engine='numexpr'):
340-
from pandas.core.computation.expressions import (
341-
_USE_NUMEXPR,
342-
_NUMEXPR_INSTALLED)
343-
344-
if engine == 'numexpr':
345-
if not _USE_NUMEXPR:
346-
import pytest
347-
pytest.skip("numexpr enabled->{enabled}, "
348-
"installed->{installed}".format(
349-
enabled=_USE_NUMEXPR,
350-
installed=_NUMEXPR_INSTALLED))
351-
352-
353339
def _skip_if_no_mock():
354340
try:
355341
import mock # noqa

0 commit comments

Comments
 (0)