From b7531bdac43898870d1752e18a6375cc8e6cc6be Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Sun, 26 Jan 2014 14:14:25 -0500 Subject: [PATCH 1/3] TST: add tests for old version of numexpr --- pandas/computation/eval.py | 12 +++++++---- pandas/computation/expressions.py | 3 ++- pandas/computation/tests/test_eval.py | 31 ++++++++++++++++++++++++++- pandas/tests/test_frame.py | 9 ++++++-- 4 files changed, 47 insertions(+), 8 deletions(-) diff --git a/pandas/computation/eval.py b/pandas/computation/eval.py index 163477b258e15..4cc68ac4770b3 100644 --- a/pandas/computation/eval.py +++ b/pandas/computation/eval.py @@ -3,13 +3,11 @@ """Top level ``eval`` module. """ -import numbers -import numpy as np from pandas.core import common as com -from pandas.compat import string_types from pandas.computation.expr import Expr, _parsers, _ensure_scope from pandas.computation.engines import _engines +from distutils.version import LooseVersion def _check_engine(engine): @@ -38,7 +36,13 @@ def _check_engine(engine): import numexpr except ImportError: raise ImportError("'numexpr' not found. Cannot use " - "engine='numexpr' if 'numexpr' is not installed") + "engine='numexpr' for query/eval " + "if 'numexpr' is not installed") + else: + ne_version = numexpr.__version__ + if ne_version < LooseVersion('2.0'): + raise ImportError("'numexpr' version is %s, " + "must be >= 2.0" % ne_version) def _check_parser(parser): diff --git a/pandas/computation/expressions.py b/pandas/computation/expressions.py index 035878e20c645..b379da9cd38bc 100644 --- a/pandas/computation/expressions.py +++ b/pandas/computation/expressions.py @@ -8,10 +8,11 @@ import numpy as np from pandas.core.common import _values_from_object +from distutils.version import LooseVersion try: import numexpr as ne - _NUMEXPR_INSTALLED = True + _NUMEXPR_INSTALLED = ne.__version__ >= LooseVersion('2.0') except ImportError: # pragma: no cover _NUMEXPR_INSTALLED = False diff --git a/pandas/computation/tests/test_eval.py b/pandas/computation/tests/test_eval.py index 6cfb8ac45312f..3d99ff92933f7 100644 --- a/pandas/computation/tests/test_eval.py +++ b/pandas/computation/tests/test_eval.py @@ -2,6 +2,7 @@ import functools from itertools import product +from distutils.version import LooseVersion import nose from nose.tools import assert_raises, assert_true, assert_false, assert_equal @@ -20,10 +21,11 @@ from pandas.computation.expressions import _USE_NUMEXPR from pandas.computation.engines import _engines from pandas.computation.expr import PythonExprVisitor, PandasExprVisitor -from pandas.computation.ops import (_binary_ops_dict, _unary_ops_dict, +from pandas.computation.ops import (_binary_ops_dict, _special_case_arith_ops_syms, _arith_ops_syms, _bool_ops_syms) from pandas.computation.common import NameResolutionError + import pandas.computation.expr as expr import pandas.util.testing as tm from pandas.util.testing import (assert_frame_equal, randbool, @@ -1553,6 +1555,33 @@ def test_name_error_exprs(): yield check_name_error_exprs, engine, parser +def check_invalid_numexpr_version(engine, parser): + def testit(): + a, b = 1, 2 + res = pd.eval('a + b', engine=engine, parser=parser) + tm.assert_equal(res, 3) + + if engine == 'numexpr': + try: + import numexpr as ne + except ImportError: + raise nose.SkipTest("no numexpr") + else: + if ne.__version__ < LooseVersion('2.0'): + with tm.assertRaisesRegexp(ImportError, "'numexpr' version is " + ".+, must be >= 2.0"): + testit() + else: + testit() + else: + testit() + + +def test_invalid_numexpr_version(): + for engine, parser in ENGINES_PARSERS: + yield check_invalid_numexpr_version, engine, parser + + if __name__ == '__main__': nose.runmodule(argv=[__file__, '-vvs', '-x', '--pdb', '--pdb-failure'], exit=False) diff --git a/pandas/tests/test_frame.py b/pandas/tests/test_frame.py index 0e30b6eda7192..0f259bf4f386f 100644 --- a/pandas/tests/test_frame.py +++ b/pandas/tests/test_frame.py @@ -11,6 +11,7 @@ import functools import itertools from itertools import product +from distutils.version import LooseVersion from pandas.compat import( map, zip, range, long, lrange, lmap, lzip, @@ -12019,8 +12020,12 @@ def skip_if_no_ne(engine='numexpr'): try: import numexpr as ne except ImportError: - raise nose.SkipTest("cannot query engine numexpr when numexpr not " - "installed") + raise nose.SkipTest("cannot query with engine numexpr when " + "numexpr not installed") + else: + if ne.__version__ < LooseVersion('2.0'): + raise nose.SkipTest("numexpr version too low: " + "%s" % ne.__version__) def skip_if_no_pandas_parser(parser): From f04f1f9228bab3bcd0136d9b0d0920aa521f5cd8 Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Sun, 26 Jan 2014 14:31:49 -0500 Subject: [PATCH 2/3] BLD/CI: add numexpr==1.4.2 to build wheels --- ci/requirements-2.6.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/requirements-2.6.txt b/ci/requirements-2.6.txt index 751d034ef97f5..8199fdd9b9648 100644 --- a/ci/requirements-2.6.txt +++ b/ci/requirements-2.6.txt @@ -5,3 +5,4 @@ pytz==2013b http://www.crummy.com/software/BeautifulSoup/bs4/download/4.2/beautifulsoup4-4.2.0.tar.gz html5lib==1.0b2 bigquery==2.0.17 +numexpr==1.4.2 From bb601b54bf25813a2c215917844f02f43d91d918 Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Sun, 26 Jan 2014 15:45:44 -0500 Subject: [PATCH 3/3] CLN/TST: DRY skip_if_no_ne --- ci/script.sh | 7 +++- pandas/computation/tests/test_eval.py | 59 ++++++++++++--------------- pandas/tests/test_frame.py | 40 +++++++----------- pandas/util/testing.py | 20 ++++++++- 4 files changed, 64 insertions(+), 62 deletions(-) diff --git a/ci/script.sh b/ci/script.sh index 0619de3d51b52..c7b9b0ce93baa 100755 --- a/ci/script.sh +++ b/ci/script.sh @@ -5,9 +5,12 @@ echo "inside $0" if [ -n "$LOCALE_OVERRIDE" ]; then export LC_ALL="$LOCALE_OVERRIDE"; echo "Setting LC_ALL to $LOCALE_OVERRIDE" + curdir="$(pwd)" + cd /tmp pycmd='import pandas; print("pandas detected console encoding: %s" % pandas.get_option("display.encoding"))' python -c "$pycmd" + cd "$curdir" fi -echo nosetests -v --exe -w /tmp -A "$NOSE_ARGS" pandas --with-xunit --xunit-file=/tmp/nosetests.xml -nosetests -v --exe -w /tmp -A "$NOSE_ARGS" pandas --with-xunit --xunit-file=/tmp/nosetests.xml +echo nosetests --exe -w /tmp -A "$NOSE_ARGS" pandas --with-xunit --xunit-file=/tmp/nosetests.xml +nosetests --exe -w /tmp -A "$NOSE_ARGS" pandas --with-xunit --xunit-file=/tmp/nosetests.xml diff --git a/pandas/computation/tests/test_eval.py b/pandas/computation/tests/test_eval.py index 3d99ff92933f7..b1cafca190bb0 100644 --- a/pandas/computation/tests/test_eval.py +++ b/pandas/computation/tests/test_eval.py @@ -18,7 +18,6 @@ from pandas.util.testing import makeCustomDataframe as mkdf from pandas.computation import pytables -from pandas.computation.expressions import _USE_NUMEXPR from pandas.computation.engines import _engines from pandas.computation.expr import PythonExprVisitor, PandasExprVisitor from pandas.computation.ops import (_binary_ops_dict, @@ -37,11 +36,6 @@ _scalar_skip = 'in', 'not in' -def skip_if_no_ne(engine='numexpr'): - if not _USE_NUMEXPR and engine == 'numexpr': - raise nose.SkipTest("numexpr engine not installed or disabled") - - def engine_has_neg_frac(engine): return _engines[engine].has_neg_frac @@ -110,7 +104,7 @@ class TestEvalNumexprPandas(tm.TestCase): @classmethod def setUpClass(cls): super(TestEvalNumexprPandas, cls).setUpClass() - skip_if_no_ne() + tm.skip_if_no_ne() import numexpr as ne cls.ne = ne cls.engine = 'numexpr' @@ -428,7 +422,7 @@ def check_single_invert_op(self, lhs, cmp1, rhs): assert_array_equal(expected, result) for engine in self.current_engines: - skip_if_no_ne(engine) + tm.skip_if_no_ne(engine) assert_array_equal(result, pd.eval('~elb', engine=engine, parser=self.parser)) @@ -459,7 +453,7 @@ def check_compound_invert_op(self, lhs, cmp1, rhs): # make sure the other engines work the same as this one for engine in self.current_engines: - skip_if_no_ne(engine) + tm.skip_if_no_ne(engine) ev = pd.eval(ex, engine=self.engine, parser=self.parser) assert_array_equal(ev, result) @@ -711,7 +705,7 @@ class TestEvalNumexprPython(TestEvalNumexprPandas): @classmethod def setUpClass(cls): super(TestEvalNumexprPython, cls).setUpClass() - skip_if_no_ne() + tm.skip_if_no_ne() import numexpr as ne cls.ne = ne cls.engine = 'numexpr' @@ -790,7 +784,7 @@ class TestAlignment(object): lhs_index_types = index_types + ('s',) # 'p' def check_align_nested_unary_op(self, engine, parser): - skip_if_no_ne(engine) + tm.skip_if_no_ne(engine) s = 'df * ~2' df = mkdf(5, 3, data_gen_f=f) res = pd.eval(s, engine=engine, parser=parser) @@ -801,7 +795,7 @@ def test_align_nested_unary_op(self): yield self.check_align_nested_unary_op, engine, parser def check_basic_frame_alignment(self, engine, parser): - skip_if_no_ne(engine) + tm.skip_if_no_ne(engine) args = product(self.lhs_index_types, self.index_types, self.index_types) for lr_idx_type, rr_idx_type, c_idx_type in args: @@ -817,7 +811,7 @@ def test_basic_frame_alignment(self): yield self.check_basic_frame_alignment, engine, parser def check_frame_comparison(self, engine, parser): - skip_if_no_ne(engine) + tm.skip_if_no_ne(engine) args = product(self.lhs_index_types, repeat=2) for r_idx_type, c_idx_type in args: df = mkdf(10, 10, data_gen_f=f, r_idx_type=r_idx_type, @@ -835,7 +829,7 @@ def test_frame_comparison(self): yield self.check_frame_comparison, engine, parser def check_medium_complex_frame_alignment(self, engine, parser): - skip_if_no_ne(engine) + tm.skip_if_no_ne(engine) args = product(self.lhs_index_types, self.index_types, self.index_types, self.index_types) @@ -852,7 +846,7 @@ def test_medium_complex_frame_alignment(self): yield self.check_medium_complex_frame_alignment, engine, parser def check_basic_frame_series_alignment(self, engine, parser): - skip_if_no_ne(engine) + tm.skip_if_no_ne(engine) def testit(r_idx_type, c_idx_type, index_name): df = mkdf(10, 10, data_gen_f=f, r_idx_type=r_idx_type, @@ -880,7 +874,7 @@ def test_basic_frame_series_alignment(self): yield self.check_basic_frame_series_alignment, engine, parser def check_basic_series_frame_alignment(self, engine, parser): - skip_if_no_ne(engine) + tm.skip_if_no_ne(engine) def testit(r_idx_type, c_idx_type, index_name): df = mkdf(10, 7, data_gen_f=f, r_idx_type=r_idx_type, @@ -913,7 +907,7 @@ def test_basic_series_frame_alignment(self): yield self.check_basic_series_frame_alignment, engine, parser def check_series_frame_commutativity(self, engine, parser): - skip_if_no_ne(engine) + tm.skip_if_no_ne(engine) args = product(self.lhs_index_types, self.index_types, ('+', '*'), ('index', 'columns')) for r_idx_type, c_idx_type, op, index_name in args: @@ -936,7 +930,7 @@ def test_series_frame_commutativity(self): yield self.check_series_frame_commutativity, engine, parser def check_complex_series_frame_alignment(self, engine, parser): - skip_if_no_ne(engine) + tm.skip_if_no_ne(engine) import random args = product(self.lhs_index_types, self.index_types, @@ -980,7 +974,7 @@ def test_complex_series_frame_alignment(self): yield self.check_complex_series_frame_alignment, engine, parser def check_performance_warning_for_poor_alignment(self, engine, parser): - skip_if_no_ne(engine) + tm.skip_if_no_ne(engine) df = DataFrame(randn(1000, 10)) s = Series(randn(10000)) if engine == 'numexpr': @@ -1036,7 +1030,7 @@ class TestOperationsNumExprPandas(tm.TestCase): @classmethod def setUpClass(cls): super(TestOperationsNumExprPandas, cls).setUpClass() - skip_if_no_ne() + tm.skip_if_no_ne() cls.engine = 'numexpr' cls.parser = 'pandas' cls.arith_ops = expr._arith_ops_syms + expr._cmp_ops_syms @@ -1196,7 +1190,7 @@ def test_assignment_fails(self): local_dict={'df': df, 'df2': df2}) def test_assignment_column(self): - skip_if_no_ne('numexpr') + tm.skip_if_no_ne('numexpr') df = DataFrame(np.random.randn(5, 2), columns=list('ab')) orig_df = df.copy() @@ -1347,10 +1341,9 @@ class TestOperationsNumExprPython(TestOperationsNumExprPandas): @classmethod def setUpClass(cls): super(TestOperationsNumExprPython, cls).setUpClass() - if not _USE_NUMEXPR: - raise nose.SkipTest("numexpr engine not installed") cls.engine = 'numexpr' cls.parser = 'python' + tm.skip_if_no_ne(cls.engine) cls.arith_ops = expr._arith_ops_syms + expr._cmp_ops_syms cls.arith_ops = filter(lambda x: x not in ('in', 'not in'), cls.arith_ops) @@ -1437,7 +1430,7 @@ def setUpClass(cls): class TestScope(object): def check_global_scope(self, e, engine, parser): - skip_if_no_ne(engine) + tm.skip_if_no_ne(engine) assert_array_equal(_var_s * 2, pd.eval(e, engine=engine, parser=parser)) @@ -1447,7 +1440,7 @@ def test_global_scope(self): yield self.check_global_scope, e, engine, parser def check_no_new_locals(self, engine, parser): - skip_if_no_ne(engine) + tm.skip_if_no_ne(engine) x = 1 lcls = locals().copy() pd.eval('x + 1', local_dict=lcls, engine=engine, parser=parser) @@ -1460,7 +1453,7 @@ def test_no_new_locals(self): yield self.check_no_new_locals, engine, parser def check_no_new_globals(self, engine, parser): - skip_if_no_ne(engine) + tm.skip_if_no_ne(engine) x = 1 gbls = globals().copy() pd.eval('x + 1', engine=engine, parser=parser) @@ -1473,21 +1466,21 @@ def test_no_new_globals(self): def test_invalid_engine(): - skip_if_no_ne() + tm.skip_if_no_ne() assertRaisesRegexp(KeyError, 'Invalid engine \'asdf\' passed', pd.eval, 'x + y', local_dict={'x': 1, 'y': 2}, engine='asdf') def test_invalid_parser(): - skip_if_no_ne() + tm.skip_if_no_ne() assertRaisesRegexp(KeyError, 'Invalid parser \'asdf\' passed', pd.eval, 'x + y', local_dict={'x': 1, 'y': 2}, parser='asdf') def check_is_expr_syntax(engine): - skip_if_no_ne(engine) + tm.skip_if_no_ne(engine) s = 1 valid1 = 's + 1' valid2 = '__y + _xx' @@ -1496,7 +1489,7 @@ def check_is_expr_syntax(engine): def check_is_expr_names(engine): - skip_if_no_ne(engine) + tm.skip_if_no_ne(engine) r, s = 1, 2 valid = 's + r' invalid = '__y + __x' @@ -1519,7 +1512,7 @@ def test_is_expr_names(): def check_disallowed_nodes(engine, parser): - skip_if_no_ne(engine) + tm.skip_if_no_ne(engine) VisitorClass = _parsers[parser] uns_ops = VisitorClass.unsupported_nodes inst = VisitorClass('x + 1', engine, parser) @@ -1534,7 +1527,7 @@ def test_disallowed_nodes(): def check_syntax_error_exprs(engine, parser): - skip_if_no_ne(engine) + tm.skip_if_no_ne(engine) e = 's +' assert_raises(SyntaxError, pd.eval, e, engine=engine, parser=parser) @@ -1545,7 +1538,7 @@ def test_syntax_error_exprs(): def check_name_error_exprs(engine, parser): - skip_if_no_ne(engine) + tm.skip_if_no_ne(engine) e = 's + t' assert_raises(NameError, pd.eval, e, engine=engine, parser=parser) diff --git a/pandas/tests/test_frame.py b/pandas/tests/test_frame.py index 0f259bf4f386f..2233f2749d3a8 100644 --- a/pandas/tests/test_frame.py +++ b/pandas/tests/test_frame.py @@ -12015,18 +12015,6 @@ def test_concat_empty_dataframe_dtypes(self): self.assertEqual(result['c'].dtype, np.float64) -def skip_if_no_ne(engine='numexpr'): - if engine == 'numexpr': - try: - import numexpr as ne - except ImportError: - raise nose.SkipTest("cannot query with engine numexpr when " - "numexpr not installed") - else: - if ne.__version__ < LooseVersion('2.0'): - raise nose.SkipTest("numexpr version too low: " - "%s" % ne.__version__) - def skip_if_no_pandas_parser(parser): if parser != 'pandas': @@ -12035,7 +12023,7 @@ def skip_if_no_pandas_parser(parser): class TestDataFrameQueryWithMultiIndex(object): def check_query_with_named_multiindex(self, parser, engine): - skip_if_no_ne(engine) + tm.skip_if_no_ne(engine) a = tm.choice(['red', 'green'], size=10) b = tm.choice(['eggs', 'ham'], size=10) index = MultiIndex.from_arrays([a, b], names=['color', 'food']) @@ -12089,7 +12077,7 @@ def test_query_with_named_multiindex(self): yield self.check_query_with_named_multiindex, parser, engine def check_query_with_unnamed_multiindex(self, parser, engine): - skip_if_no_ne(engine) + tm.skip_if_no_ne(engine) a = tm.choice(['red', 'green'], size=10) b = tm.choice(['eggs', 'ham'], size=10) index = MultiIndex.from_arrays([a, b]) @@ -12181,7 +12169,7 @@ def test_query_with_unnamed_multiindex(self): yield self.check_query_with_unnamed_multiindex, parser, engine def check_query_with_partially_named_multiindex(self, parser, engine): - skip_if_no_ne(engine) + tm.skip_if_no_ne(engine) a = tm.choice(['red', 'green'], size=10) b = np.arange(10) index = MultiIndex.from_arrays([a, b]) @@ -12247,7 +12235,7 @@ def test_raise_on_panel_with_multiindex(self): yield self.check_raise_on_panel_with_multiindex, parser, engine def check_raise_on_panel_with_multiindex(self, parser, engine): - skip_if_no_ne() + tm.skip_if_no_ne() p = tm.makePanel(7) p.items = tm.makeCustomIndex(len(p.items), nlevels=2) with tm.assertRaises(NotImplementedError): @@ -12258,7 +12246,7 @@ def test_raise_on_panel4d_with_multiindex(self): yield self.check_raise_on_panel4d_with_multiindex, parser, engine def check_raise_on_panel4d_with_multiindex(self, parser, engine): - skip_if_no_ne() + tm.skip_if_no_ne() p4d = tm.makePanel4D(7) p4d.items = tm.makeCustomIndex(len(p4d.items), nlevels=2) with tm.assertRaises(NotImplementedError): @@ -12272,7 +12260,7 @@ def setUpClass(cls): super(TestDataFrameQueryNumExprPandas, cls).setUpClass() cls.engine = 'numexpr' cls.parser = 'pandas' - skip_if_no_ne() + tm.skip_if_no_ne() @classmethod def tearDownClass(cls): @@ -12501,7 +12489,7 @@ def setUpClass(cls): super(TestDataFrameQueryNumExprPython, cls).setUpClass() cls.engine = 'numexpr' cls.parser = 'python' - skip_if_no_ne(cls.engine) + tm.skip_if_no_ne(cls.engine) cls.frame = _frame.copy() def test_date_query_method(self): @@ -12620,7 +12608,7 @@ def setUpClass(cls): class TestDataFrameQueryStrings(object): def check_str_query_method(self, parser, engine): - skip_if_no_ne(engine) + tm.skip_if_no_ne(engine) df = DataFrame(randn(10, 1), columns=['b']) df['strings'] = Series(list('aabbccddee')) expect = df[df.strings == 'a'] @@ -12664,7 +12652,7 @@ def test_str_list_query_method(self): yield self.check_str_list_query_method, parser, engine def check_str_list_query_method(self, parser, engine): - skip_if_no_ne(engine) + tm.skip_if_no_ne(engine) df = DataFrame(randn(10, 1), columns=['b']) df['strings'] = Series(list('aabbccddee')) expect = df[df.strings.isin(['a', 'b'])] @@ -12703,7 +12691,7 @@ def check_str_list_query_method(self, parser, engine): assert_frame_equal(res, expect) def check_query_with_string_columns(self, parser, engine): - skip_if_no_ne(engine) + tm.skip_if_no_ne(engine) df = DataFrame({'a': list('aaaabbbbcccc'), 'b': list('aabbccddeeff'), 'c': np.random.randint(5, size=12), @@ -12728,7 +12716,7 @@ def test_query_with_string_columns(self): yield self.check_query_with_string_columns, parser, engine def check_object_array_eq_ne(self, parser, engine): - skip_if_no_ne(engine) + tm.skip_if_no_ne(engine) df = DataFrame({'a': list('aaaabbbbcccc'), 'b': list('aabbccddeeff'), 'c': np.random.randint(5, size=12), @@ -12746,7 +12734,7 @@ def test_object_array_eq_ne(self): yield self.check_object_array_eq_ne, parser, engine def check_query_with_nested_strings(self, parser, engine): - skip_if_no_ne(engine) + tm.skip_if_no_ne(engine) skip_if_no_pandas_parser(parser) from pandas.compat import StringIO raw = """id event timestamp @@ -12781,7 +12769,7 @@ def setUpClass(cls): super(TestDataFrameEvalNumExprPandas, cls).setUpClass() cls.engine = 'numexpr' cls.parser = 'pandas' - skip_if_no_ne() + tm.skip_if_no_ne() def setUp(self): self.frame = DataFrame(randn(10, 3), columns=list('abc')) @@ -12808,7 +12796,7 @@ def setUpClass(cls): super(TestDataFrameEvalNumExprPython, cls).setUpClass() cls.engine = 'numexpr' cls.parser = 'python' - skip_if_no_ne() + tm.skip_if_no_ne(cls.engine) class TestDataFrameEvalPythonPandas(TestDataFrameEvalNumExprPandas): diff --git a/pandas/util/testing.py b/pandas/util/testing.py index 80e33eb1717da..a3bd88ed2ab4a 100644 --- a/pandas/util/testing.py +++ b/pandas/util/testing.py @@ -22,7 +22,7 @@ import numpy as np import pandas as pd -from pandas.core.common import isnull, _is_sequence +from pandas.core.common import _is_sequence import pandas.core.index as index import pandas.core.series as series import pandas.core.frame as frame @@ -1391,3 +1391,21 @@ def assert_produces_warning(expected_warning=Warning, filter_level="always"): % expected_warning.__name__) assert not extra_warnings, ("Caused unexpected warning(s): %r." % extra_warnings) + + +def skip_if_no_ne(engine='numexpr'): + import nose + _USE_NUMEXPR = pd.computation.expressions._USE_NUMEXPR + + if engine == 'numexpr': + try: + import numexpr as ne + except ImportError: + raise nose.SkipTest("numexpr not installed") + + if not _USE_NUMEXPR: + raise nose.SkipTest("numexpr disabled") + + if ne.__version__ < LooseVersion('2.0'): + raise nose.SkipTest("numexpr version too low: " + "%s" % ne.__version__)