diff --git a/pandas/tests/computation/test_eval.py b/pandas/tests/computation/test_eval.py index 06e645563d51c..90f197738543a 100644 --- a/pandas/tests/computation/test_eval.py +++ b/pandas/tests/computation/test_eval.py @@ -97,11 +97,11 @@ def _is_py3_complex_incompat(result, expected): _good_arith_ops = com.difference(_arith_ops_syms, _special_case_arith_ops_syms) +@td.skip_if_no_ne class TestEvalNumexprPandas(object): @classmethod def setup_class(cls): - tm.skip_if_no_ne() import numexpr as ne cls.ne = ne cls.engine = 'numexpr' @@ -374,7 +374,6 @@ def check_single_invert_op(self, lhs, cmp1, rhs): tm.assert_almost_equal(expected, result) for engine in self.current_engines: - tm.skip_if_no_ne(engine) tm.assert_almost_equal(result, pd.eval('~elb', engine=engine, parser=self.parser)) @@ -400,7 +399,6 @@ def check_compound_invert_op(self, lhs, cmp1, rhs): # make sure the other engines work the same as this one for engine in self.current_engines: - tm.skip_if_no_ne(engine) ev = pd.eval(ex, engine=self.engine, parser=self.parser) tm.assert_almost_equal(ev, result) @@ -731,12 +729,12 @@ def test_disallow_python_keywords(self): df.query('lambda == 0') +@td.skip_if_no_ne class TestEvalNumexprPython(TestEvalNumexprPandas): @classmethod def setup_class(cls): super(TestEvalNumexprPython, cls).setup_class() - tm.skip_if_no_ne() import numexpr as ne cls.ne = ne cls.engine = 'numexpr' @@ -1078,11 +1076,11 @@ def test_performance_warning_for_poor_alignment(self, engine, parser): # ------------------------------------ # Slightly more complex ops +@td.skip_if_no_ne class TestOperationsNumExprPandas(object): @classmethod def setup_class(cls): - tm.skip_if_no_ne() cls.engine = 'numexpr' cls.parser = 'pandas' cls.arith_ops = expr._arith_ops_syms + expr._cmp_ops_syms @@ -1528,6 +1526,7 @@ def test_simple_in_ops(self): parser=self.parser) +@td.skip_if_no_ne class TestOperationsNumExprPython(TestOperationsNumExprPandas): @classmethod @@ -1535,7 +1534,6 @@ def setup_class(cls): super(TestOperationsNumExprPython, cls).setup_class() cls.engine = 'numexpr' cls.parser = 'python' - tm.skip_if_no_ne(cls.engine) cls.arith_ops = expr._arith_ops_syms + expr._cmp_ops_syms cls.arith_ops = filter(lambda x: x not in ('in', 'not in'), cls.arith_ops) @@ -1623,11 +1621,11 @@ def setup_class(cls): cls.arith_ops = expr._arith_ops_syms + expr._cmp_ops_syms +@td.skip_if_no_ne class TestMathPythonPython(object): @classmethod def setup_class(cls): - tm.skip_if_no_ne() cls.engine = 'python' cls.parser = 'pandas' cls.unary_fns = _unary_math_ops @@ -1782,15 +1780,15 @@ def test_no_new_globals(self, engine, parser): assert gbls == gbls2 +@td.skip_if_no_ne def test_invalid_engine(): - tm.skip_if_no_ne() tm.assert_raises_regex(KeyError, 'Invalid engine \'asdf\' passed', pd.eval, 'x + y', local_dict={'x': 1, 'y': 2}, engine='asdf') +@td.skip_if_no_ne def test_invalid_parser(): - tm.skip_if_no_ne() tm.assert_raises_regex(KeyError, 'Invalid parser \'asdf\' passed', pd.eval, 'x + y', local_dict={'x': 1, 'y': 2}, parser='asdf') @@ -1803,7 +1801,6 @@ def test_invalid_parser(): @pytest.mark.parametrize('engine', _parsers) @pytest.mark.parametrize('parser', _parsers) def test_disallowed_nodes(engine, parser): - tm.skip_if_no_ne(engine) VisitorClass = _parsers[parser] uns_ops = VisitorClass.unsupported_nodes inst = VisitorClass('x + 1', engine, parser) diff --git a/pandas/tests/frame/test_query_eval.py b/pandas/tests/frame/test_query_eval.py index a6c36792ef074..22066d59cf14d 100644 --- a/pandas/tests/frame/test_query_eval.py +++ b/pandas/tests/frame/test_query_eval.py @@ -17,13 +17,14 @@ makeCustomDataframe as mkdf) import pandas.util.testing as tm +import pandas.util._test_decorators as td from pandas.core.computation.check import _NUMEXPR_INSTALLED from pandas.tests.frame.common import TestData PARSERS = 'python', 'pandas' -ENGINES = 'python', 'numexpr' +ENGINES = 'python', pytest.param('numexpr', marks=td.skip_if_no_ne) @pytest.fixture(params=PARSERS, ids=lambda x: x) @@ -41,13 +42,6 @@ def skip_if_no_pandas_parser(parser): pytest.skip("cannot evaluate with parser {0!r}".format(parser)) -def skip_if_no_ne(engine='numexpr'): - if engine == 'numexpr': - if not _NUMEXPR_INSTALLED: - pytest.skip("cannot query engine numexpr when numexpr not " - "installed") - - class TestCompat(object): def setup_method(self, method): @@ -175,7 +169,6 @@ def test_eval_resolvers_as_list(self): class TestDataFrameQueryWithMultiIndex(object): def test_query_with_named_multiindex(self, parser, engine): - tm.skip_if_no_ne(engine) skip_if_no_pandas_parser(parser) a = np.random.choice(['red', 'green'], size=10) b = np.random.choice(['eggs', 'ham'], size=10) @@ -225,7 +218,6 @@ def test_query_with_named_multiindex(self, parser, engine): assert_frame_equal(res2, exp) def test_query_with_unnamed_multiindex(self, parser, engine): - tm.skip_if_no_ne(engine) skip_if_no_pandas_parser(parser) a = np.random.choice(['red', 'green'], size=10) b = np.random.choice(['eggs', 'ham'], size=10) @@ -316,7 +308,6 @@ def test_query_with_unnamed_multiindex(self, parser, engine): assert_frame_equal(res2, exp) def test_query_with_partially_named_multiindex(self, parser, engine): - tm.skip_if_no_ne(engine) skip_if_no_pandas_parser(parser) a = np.random.choice(['red', 'green'], size=10) b = np.arange(10) @@ -370,27 +361,25 @@ def to_series(mi, level): raise AssertionError("object must be a Series or Index") def test_raise_on_panel_with_multiindex(self, parser, engine): - tm.skip_if_no_ne() p = tm.makePanel(7) p.items = tm.makeCustomIndex(len(p.items), nlevels=2) with pytest.raises(NotImplementedError): pd.eval('p + 1', parser=parser, engine=engine) def test_raise_on_panel4d_with_multiindex(self, parser, engine): - tm.skip_if_no_ne() p4d = tm.makePanel4D(7) p4d.items = tm.makeCustomIndex(len(p4d.items), nlevels=2) with pytest.raises(NotImplementedError): pd.eval('p4d + 1', parser=parser, engine=engine) +@td.skip_if_no_ne class TestDataFrameQueryNumExprPandas(object): @classmethod def setup_class(cls): cls.engine = 'numexpr' cls.parser = 'pandas' - tm.skip_if_no_ne(cls.engine) @classmethod def teardown_class(cls): @@ -714,6 +703,7 @@ def test_inf(self): assert_frame_equal(result, expected) +@td.skip_if_no_ne class TestDataFrameQueryNumExprPython(TestDataFrameQueryNumExprPandas): @classmethod @@ -721,7 +711,6 @@ def setup_class(cls): super(TestDataFrameQueryNumExprPython, cls).setup_class() cls.engine = 'numexpr' cls.parser = 'python' - tm.skip_if_no_ne(cls.engine) cls.frame = TestData().frame def test_date_query_no_attribute_access(self): @@ -859,7 +848,6 @@ def test_query_builtin(self): class TestDataFrameQueryStrings(object): def test_str_query_method(self, parser, engine): - tm.skip_if_no_ne(engine) df = DataFrame(randn(10, 1), columns=['b']) df['strings'] = Series(list('aabbccddee')) expect = df[df.strings == 'a'] @@ -896,7 +884,6 @@ def test_str_query_method(self, parser, engine): assert_frame_equal(res, df[~df.strings.isin(['a'])]) def test_str_list_query_method(self, parser, engine): - tm.skip_if_no_ne(engine) df = DataFrame(randn(10, 1), columns=['b']) df['strings'] = Series(list('aabbccddee')) expect = df[df.strings.isin(['a', 'b'])] @@ -935,7 +922,6 @@ def test_str_list_query_method(self, parser, engine): assert_frame_equal(res, expect) def test_query_with_string_columns(self, parser, engine): - tm.skip_if_no_ne(engine) df = DataFrame({'a': list('aaaabbbbcccc'), 'b': list('aabbccddeeff'), 'c': np.random.randint(5, size=12), @@ -956,7 +942,6 @@ def test_query_with_string_columns(self, parser, engine): df.query('a in b and c < d', parser=parser, engine=engine) def test_object_array_eq_ne(self, parser, engine): - tm.skip_if_no_ne(engine) df = DataFrame({'a': list('aaaabbbbcccc'), 'b': list('aabbccddeeff'), 'c': np.random.randint(5, size=12), @@ -970,7 +955,6 @@ def test_object_array_eq_ne(self, parser, engine): assert_frame_equal(res, exp) def test_query_with_nested_strings(self, parser, engine): - tm.skip_if_no_ne(engine) skip_if_no_pandas_parser(parser) raw = """id event timestamp 1 "page 1 load" 1/1/2014 0:00:01 @@ -995,7 +979,6 @@ def test_query_with_nested_strings(self, parser, engine): def test_query_with_nested_special_character(self, parser, engine): skip_if_no_pandas_parser(parser) - tm.skip_if_no_ne(engine) df = DataFrame({'a': ['a', 'b', 'test & test'], 'b': [1, 2, 3]}) res = df.query('a == "test & test"', parser=parser, engine=engine) @@ -1003,7 +986,6 @@ def test_query_with_nested_special_character(self, parser, engine): assert_frame_equal(res, expec) def test_query_lex_compare_strings(self, parser, engine): - tm.skip_if_no_ne(engine=engine) import operator as opr a = Series(np.random.choice(list('abcde'), 20)) @@ -1018,7 +1000,6 @@ def test_query_lex_compare_strings(self, parser, engine): assert_frame_equal(res, expected) def test_query_single_element_booleans(self, parser, engine): - tm.skip_if_no_ne(engine) columns = 'bid', 'bidsize', 'ask', 'asksize' data = np.random.randint(2, size=(1, len(columns))).astype(bool) df = DataFrame(data, columns=columns) @@ -1027,7 +1008,6 @@ def test_query_single_element_booleans(self, parser, engine): assert_frame_equal(res, expected) def test_query_string_scalar_variable(self, parser, engine): - tm.skip_if_no_ne(engine) skip_if_no_pandas_parser(parser) df = pd.DataFrame({'Symbol': ['BUD US', 'BUD US', 'IBM US', 'IBM US'], 'Price': [109.70, 109.72, 183.30, 183.35]}) @@ -1037,13 +1017,7 @@ def test_query_string_scalar_variable(self, parser, engine): assert_frame_equal(e, r) -class TestDataFrameEvalNumExprPandas(object): - - @classmethod - def setup_class(cls): - cls.engine = 'numexpr' - cls.parser = 'pandas' - tm.skip_if_no_ne() +class TestDataFrameEvalWithFrame(object): def setup_method(self, method): self.frame = DataFrame(randn(10, 3), columns=list('abc')) @@ -1051,49 +1025,21 @@ def setup_method(self, method): def teardown_method(self, method): del self.frame - def test_simple_expr(self): - res = self.frame.eval('a + b', engine=self.engine, parser=self.parser) + def test_simple_expr(self, parser, engine): + res = self.frame.eval('a + b', engine=engine, parser=parser) expect = self.frame.a + self.frame.b assert_series_equal(res, expect) - def test_bool_arith_expr(self): - res = self.frame.eval('a[a < 1] + b', engine=self.engine, - parser=self.parser) + def test_bool_arith_expr(self, parser, engine): + res = self.frame.eval('a[a < 1] + b', engine=engine, parser=parser) expect = self.frame.a[self.frame.a < 1] + self.frame.b assert_series_equal(res, expect) - def test_invalid_type_for_operator_raises(self): + def test_invalid_type_for_operator_raises(self, parser, engine): df = DataFrame({'a': [1, 2], 'b': ['c', 'd']}) ops = '+', '-', '*', '/' for op in ops: with tm.assert_raises_regex(TypeError, "unsupported operand type\(s\) " "for .+: '.+' and '.+'"): - df.eval('a {0} b'.format(op), engine=self.engine, - parser=self.parser) - - -class TestDataFrameEvalNumExprPython(TestDataFrameEvalNumExprPandas): - - @classmethod - def setup_class(cls): - super(TestDataFrameEvalNumExprPython, cls).setup_class() - cls.engine = 'numexpr' - cls.parser = 'python' - tm.skip_if_no_ne(cls.engine) - - -class TestDataFrameEvalPythonPandas(TestDataFrameEvalNumExprPandas): - - @classmethod - def setup_class(cls): - super(TestDataFrameEvalPythonPandas, cls).setup_class() - cls.engine = 'python' - cls.parser = 'pandas' - - -class TestDataFrameEvalPythonPython(TestDataFrameEvalNumExprPython): - - @classmethod - def setup_class(cls): - cls.engine = cls.parser = 'python' + df.eval('a {0} b'.format(op), engine=engine, parser=parser) diff --git a/pandas/tests/frame/test_to_csv.py b/pandas/tests/frame/test_to_csv.py index ca8a0d8bda3ab..0ca25735fc03f 100644 --- a/pandas/tests/frame/test_to_csv.py +++ b/pandas/tests/frame/test_to_csv.py @@ -21,6 +21,7 @@ ensure_clean, makeCustomDataframe as mkdf) import pandas.util.testing as tm +import pandas.util._test_decorators as td from pandas.tests.frame.common import TestData @@ -965,10 +966,10 @@ def test_to_csv_compression_bz2(self): for col in df.columns: assert col in text + @td.skip_if_no_lzma def test_to_csv_compression_xz(self): # GH11852 # use the compression kw in to_csv - tm._skip_if_no_lzma() df = DataFrame([[0.123456, 0.234567, 0.567567], [12.32112, 123123.2, 321321.2]], index=['A', 'B'], columns=['X', 'Y', 'Z']) diff --git a/pandas/tests/io/json/test_compression.py b/pandas/tests/io/json/test_compression.py index 077752039a558..a83ec53904b28 100644 --- a/pandas/tests/io/json/test_compression.py +++ b/pandas/tests/io/json/test_compression.py @@ -3,10 +3,12 @@ import pandas as pd from pandas import compat import pandas.util.testing as tm +import pandas.util._test_decorators as td from pandas.util.testing import assert_frame_equal, assert_raises_regex -COMPRESSION_TYPES = [None, 'bz2', 'gzip', 'xz'] +COMPRESSION_TYPES = [None, 'bz2', 'gzip', + pytest.param('xz', marks=td.skip_if_no_lzma)] def decompress_file(path, compression): @@ -32,9 +34,6 @@ def decompress_file(path, compression): @pytest.mark.parametrize('compression', COMPRESSION_TYPES) def test_compression_roundtrip(compression): - if compression == 'xz': - tm._skip_if_no_lzma() - df = pd.DataFrame([[0.123456, 0.234567, 0.567567], [12.32112, 123123.2, 321321.2]], index=['A', 'B'], columns=['X', 'Y', 'Z']) @@ -74,9 +73,6 @@ def test_with_s3_url(compression): pytest.importorskip('s3fs') moto = pytest.importorskip('moto') - if compression == 'xz': - tm._skip_if_no_lzma() - df = pd.read_json('{"a": [1, 2, 3], "b": [4, 5, 6]}') with moto.mock_s3(): conn = boto3.resource("s3", region_name="us-east-1") @@ -94,9 +90,6 @@ def test_with_s3_url(compression): @pytest.mark.parametrize('compression', COMPRESSION_TYPES) def test_lines_with_compression(compression): - if compression == 'xz': - tm._skip_if_no_lzma() - with tm.ensure_clean() as path: df = pd.read_json('{"a": [1, 2, 3], "b": [4, 5, 6]}') df.to_json(path, orient='records', lines=True, compression=compression) @@ -107,9 +100,6 @@ def test_lines_with_compression(compression): @pytest.mark.parametrize('compression', COMPRESSION_TYPES) def test_chunksize_with_compression(compression): - if compression == 'xz': - tm._skip_if_no_lzma() - with tm.ensure_clean() as path: df = pd.read_json('{"a": ["foo", "bar", "baz"], "b": [4, 5, 6]}') df.to_json(path, orient='records', lines=True, compression=compression) diff --git a/pandas/tests/io/parser/compression.py b/pandas/tests/io/parser/compression.py index 84db9d14eee07..4291d59123e8b 100644 --- a/pandas/tests/io/parser/compression.py +++ b/pandas/tests/io/parser/compression.py @@ -8,7 +8,9 @@ import pytest import pandas as pd +import pandas.compat as compat import pandas.util.testing as tm +import pandas.util._test_decorators as td class CompressionTests(object): @@ -117,8 +119,9 @@ def test_bz2(self): result = self.read_csv(path, compression='infer') tm.assert_frame_equal(result, expected) + @td.skip_if_no_lzma def test_xz(self): - lzma = tm._skip_if_no_lzma() + lzma = compat.import_lzma() with open(self.csv1, 'rb') as data_file: data = data_file.read() diff --git a/pandas/tests/io/parser/test_network.py b/pandas/tests/io/parser/test_network.py index d00d3f31ce189..2d0a23d71a2e6 100644 --- a/pandas/tests/io/parser/test_network.py +++ b/pandas/tests/io/parser/test_network.py @@ -7,6 +7,7 @@ import pytest import pandas.util.testing as tm +import pandas.util._test_decorators as td from pandas import DataFrame from pandas.io.parsers import read_csv, read_table from pandas.compat import BytesIO @@ -14,12 +15,11 @@ @pytest.mark.network @pytest.mark.parametrize( - "compression,extension", - [('gzip', '.gz'), ('bz2', '.bz2'), ('zip', '.zip'), - pytest.param('xz', '.xz', - marks=pytest.mark.skipif(not tm._check_if_lzma(), - reason='need backports.lzma ' - 'to run'))]) + "compression,extension", [ + ('gzip', '.gz'), ('bz2', '.bz2'), ('zip', '.zip'), + pytest.param('xz', '.xz', marks=td.skip_if_no_lzma) + ] +) @pytest.mark.parametrize('mode', ['explicit', 'infer']) @pytest.mark.parametrize('engine', ['python', 'c']) def test_compressed_urls(salaries_table, compression, extension, mode, engine): diff --git a/pandas/tests/io/test_pickle.py b/pandas/tests/io/test_pickle.py index c49339f112d6a..d5bcf72488d09 100644 --- a/pandas/tests/io/test_pickle.py +++ b/pandas/tests/io/test_pickle.py @@ -23,6 +23,7 @@ from pandas.compat import is_platform_little_endian import pandas import pandas.util.testing as tm +import pandas.util._test_decorators as td from pandas.tseries.offsets import Day, MonthEnd import shutil import sys @@ -382,12 +383,11 @@ def decompress_file(self, src_path, dest_path, compression): fh.write(f.read()) f.close() - @pytest.mark.parametrize('compression', [None, 'gzip', 'bz2', 'xz']) + @pytest.mark.parametrize('compression', [ + None, 'gzip', 'bz2', + pytest.param('xz', marks=td.skip_if_no_lzma) # issue 11666 + ]) def test_write_explicit(self, compression, get_random_path): - # issue 11666 - if compression == 'xz': - tm._skip_if_no_lzma() - base = get_random_path path1 = base + ".compressed" path2 = base + ".raw" @@ -414,11 +414,11 @@ def test_write_explicit_bad(self, compression, get_random_path): df = tm.makeDataFrame() df.to_pickle(path, compression=compression) - @pytest.mark.parametrize('ext', ['', '.gz', '.bz2', '.xz', '.no_compress']) + @pytest.mark.parametrize('ext', [ + '', '.gz', '.bz2', '.no_compress', + pytest.param('.xz', marks=td.skip_if_no_lzma) + ]) def test_write_infer(self, ext, get_random_path): - if ext == '.xz': - tm._skip_if_no_lzma() - base = get_random_path path1 = base + ext path2 = base + ".raw" @@ -442,12 +442,11 @@ def test_write_infer(self, ext, get_random_path): tm.assert_frame_equal(df, df2) - @pytest.mark.parametrize('compression', [None, 'gzip', 'bz2', 'xz', "zip"]) + @pytest.mark.parametrize('compression', [ + None, 'gzip', 'bz2', "zip", + pytest.param('xz', marks=td.skip_if_no_lzma) + ]) def test_read_explicit(self, compression, get_random_path): - # issue 11666 - if compression == 'xz': - tm._skip_if_no_lzma() - base = get_random_path path1 = base + ".raw" path2 = base + ".compressed" @@ -466,12 +465,11 @@ def test_read_explicit(self, compression, get_random_path): tm.assert_frame_equal(df, df2) - @pytest.mark.parametrize('ext', ['', '.gz', '.bz2', '.xz', '.zip', - '.no_compress']) + @pytest.mark.parametrize('ext', [ + '', '.gz', '.bz2', '.zip', '.no_compress', + pytest.param('.xz', marks=td.skip_if_no_lzma) + ]) def test_read_infer(self, ext, get_random_path): - if ext == '.xz': - tm._skip_if_no_lzma() - base = get_random_path path1 = base + ".raw" path2 = base + ext diff --git a/pandas/util/_test_decorators.py b/pandas/util/_test_decorators.py index 95a9a8fed42f7..0fd5648739e5c 100644 --- a/pandas/util/_test_decorators.py +++ b/pandas/util/_test_decorators.py @@ -28,7 +28,10 @@ def test_foo(): import locale from distutils.version import LooseVersion -from pandas.compat import is_platform_windows, is_platform_32bit, PY3 +from pandas.compat import (is_platform_windows, is_platform_32bit, PY3, + import_lzma) +from pandas.core.computation.expressions import (_USE_NUMEXPR, + _NUMEXPR_INSTALLED) def safe_import(mod_name, min_version=None): @@ -99,6 +102,13 @@ def _skip_if_no_scipy(): safe_import('scipy.interpolate')) +def _skip_if_no_lzma(): + try: + import_lzma() + except ImportError: + return True + + def skip_if_no(package, min_version=None): """ Generic function to help skip test functions when required packages are not @@ -153,3 +163,10 @@ def decorated_func(func): lang=locale.getlocale()[0])) skip_if_no_scipy = pytest.mark.skipif(_skip_if_no_scipy(), reason="Missing SciPy requirement") +skip_if_no_lzma = pytest.mark.skipif(_skip_if_no_lzma(), + reason="need backports.lzma to run") +skip_if_no_ne = pytest.mark.skipif(not _USE_NUMEXPR, + reason="numexpr enabled->{enabled}, " + "installed->{installed}".format( + enabled=_USE_NUMEXPR, + installed=_NUMEXPR_INSTALLED)) diff --git a/pandas/util/testing.py b/pandas/util/testing.py index 4c6e3217ed6b4..131d470053a79 100644 --- a/pandas/util/testing.py +++ b/pandas/util/testing.py @@ -336,32 +336,6 @@ def _skip_if_no_scipy(): pytest.importorskip("scipy.interpolate") -def _check_if_lzma(): - try: - return compat.import_lzma() - except ImportError: - return False - - -def _skip_if_no_lzma(): - import pytest - return _check_if_lzma() or pytest.skip('need backports.lzma to run') - - -def skip_if_no_ne(engine='numexpr'): - from pandas.core.computation.expressions import ( - _USE_NUMEXPR, - _NUMEXPR_INSTALLED) - - if engine == 'numexpr': - if not _USE_NUMEXPR: - import pytest - pytest.skip("numexpr enabled->{enabled}, " - "installed->{installed}".format( - enabled=_USE_NUMEXPR, - installed=_NUMEXPR_INSTALLED)) - - def _skip_if_no_mock(): try: import mock # noqa