From bad7c0f4379d07d6a6f335b7052b2145e62bd0b4 Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Wed, 2 Mar 2016 08:35:24 -0500 Subject: [PATCH 1/2] COMPAT: blacklist numexpr=2.4.4 closes #12489 closes #12511 --- ci/requirements-3.4_SLOW.run | 2 +- doc/source/install.rst | 2 +- doc/source/whatsnew/v0.18.0.txt | 4 ++ pandas/computation/__init__.py | 30 +++++++++++ pandas/computation/eval.py | 14 ++--- pandas/computation/expressions.py | 15 +----- pandas/computation/tests/test_compat.py | 69 +++++++++++++++++++++++++ pandas/computation/tests/test_eval.py | 27 ---------- 8 files changed, 110 insertions(+), 53 deletions(-) create mode 100644 pandas/computation/tests/test_compat.py diff --git a/ci/requirements-3.4_SLOW.run b/ci/requirements-3.4_SLOW.run index f0101d34204a3..f9f226e3f1465 100644 --- a/ci/requirements-3.4_SLOW.run +++ b/ci/requirements-3.4_SLOW.run @@ -9,7 +9,7 @@ html5lib patsy beautiful-soup scipy -numexpr +numexpr=2.4.4 pytables matplotlib lxml diff --git a/doc/source/install.rst b/doc/source/install.rst index 3836180af520f..11b9115aa9c81 100644 --- a/doc/source/install.rst +++ b/doc/source/install.rst @@ -225,7 +225,7 @@ Recommended Dependencies * `numexpr `__: for accelerating certain numerical operations. ``numexpr`` uses multiple cores as well as smart chunking and caching to achieve large speedups. - If installed, must be Version 2.1 or higher. Version 2.4.6 or higher on Windows is highly recommended. + If installed, must be Version 2.1 or higher (excluding a buggy 2.4.4). Version 2.4.6 or higher is highly recommended. * `bottleneck `__: for accelerating certain types of ``nan`` evaluations. ``bottleneck`` uses specialized cython routines to achieve large speedups. diff --git a/doc/source/whatsnew/v0.18.0.txt b/doc/source/whatsnew/v0.18.0.txt index 34e024def332b..049152dba9a30 100644 --- a/doc/source/whatsnew/v0.18.0.txt +++ b/doc/source/whatsnew/v0.18.0.txt @@ -12,6 +12,10 @@ users upgrade to this version. pandas >= 0.18.0 no longer supports compatibility with Python version 2.6 and 3.3 (:issue:`7718`, :issue:`11273`) +.. warning:: + + ``numexpr`` version 2.4.4 will now show a warning and not be used as a computation back-end for pandas because of some buggy behavior. This does not affect other versions (>= 2.1 and >= 2.4.6). (:issue:`12489`) + Highlights include: - Moving and expanding window functions are now methods on Series and DataFrame, diff --git a/pandas/computation/__init__.py b/pandas/computation/__init__.py index e69de29bb2d1d..9e94215eecf62 100644 --- a/pandas/computation/__init__.py +++ b/pandas/computation/__init__.py @@ -0,0 +1,30 @@ + +import warnings +from distutils.version import LooseVersion + +_NUMEXPR_INSTALLED = False + +try: + import numexpr as ne + ver = ne.__version__ + _NUMEXPR_INSTALLED = ver >= LooseVersion('2.1') + + # we specifically disallow 2.4.4 as + # has some hard-to-diagnose bugs + if ver == LooseVersion('2.4.4'): + _NUMEXPR_INSTALLED = False + warnings.warn( + "The installed version of numexpr {ver} is not supported " + "in pandas and will be not be used\n".format(ver=ver), + UserWarning) + + elif not _NUMEXPR_INSTALLED: + warnings.warn( + "The installed version of numexpr {ver} is not supported " + "in pandas and will be not be used\nThe minimum supported " + "version is 2.1\n".format(ver=ver), UserWarning) + +except ImportError: # pragma: no cover + pass + +__all__ = ['_NUMEXPR_INSTALLED'] diff --git a/pandas/computation/eval.py b/pandas/computation/eval.py index d2d16acc27fb6..c3300ffca468e 100644 --- a/pandas/computation/eval.py +++ b/pandas/computation/eval.py @@ -6,11 +6,11 @@ import warnings import tokenize from pandas.core import common as com +from pandas.computation import _NUMEXPR_INSTALLED from pandas.computation.expr import Expr, _parsers, tokenize_string from pandas.computation.scope import _ensure_scope from pandas.compat import string_types from pandas.computation.engines import _engines -from distutils.version import LooseVersion def _check_engine(engine): @@ -35,17 +35,11 @@ def _check_engine(engine): # that won't necessarily be import-able) # Could potentially be done on engine instantiation if engine == 'numexpr': - try: - import numexpr - except ImportError: - raise ImportError("'numexpr' not found. Cannot use " + if not _NUMEXPR_INSTALLED: + raise ImportError("'numexpr' is not installed or an " + "unsupported version. Cannot use " "engine='numexpr' for query/eval " "if 'numexpr' is not installed") - else: - ne_version = numexpr.__version__ - if ne_version < LooseVersion('2.1'): - raise ImportError("'numexpr' version is %s, " - "must be >= 2.1" % ne_version) def _check_parser(parser): diff --git a/pandas/computation/expressions.py b/pandas/computation/expressions.py index 6e33250010c2b..82fbefd5ab608 100644 --- a/pandas/computation/expressions.py +++ b/pandas/computation/expressions.py @@ -9,20 +9,7 @@ import warnings import numpy as np from pandas.core.common import _values_from_object -from distutils.version import LooseVersion - -try: - import numexpr as ne - ver = ne.__version__ - _NUMEXPR_INSTALLED = ver >= LooseVersion('2.1') - if not _NUMEXPR_INSTALLED: - warnings.warn( - "The installed version of numexpr {ver} is not supported " - "in pandas and will be not be used\nThe minimum supported " - "version is 2.1\n".format(ver=ver), UserWarning) - -except ImportError: # pragma: no cover - _NUMEXPR_INSTALLED = False +from pandas.computation import _NUMEXPR_INSTALLED _TEST_MODE = None _TEST_RESULT = None diff --git a/pandas/computation/tests/test_compat.py b/pandas/computation/tests/test_compat.py new file mode 100644 index 0000000000000..80b415739c647 --- /dev/null +++ b/pandas/computation/tests/test_compat.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python + +# flake8: noqa + +import nose +from itertools import product +from distutils.version import LooseVersion + +import pandas as pd +from pandas.util import testing as tm + +from pandas.computation.engines import _engines +import pandas.computation.expr as expr + +ENGINES_PARSERS = list(product(_engines, expr._parsers)) + + +def test_compat(): + # test we have compat with our version of nu + + from pandas.computation import _NUMEXPR_INSTALLED + try: + import numexpr as ne + ver = ne.__version__ + if ver == LooseVersion('2.4.4'): + assert not _NUMEXPR_INSTALLED + elif ver < LooseVersion('2.1'): + with tm.assert_produces_warning(UserWarning, + check_stacklevel=False): + assert not _NUMEXPR_INSTALLED + else: + assert _NUMEXPR_INSTALLED + + except ImportError: + raise nose.SkipTest("not testing numexpr version compat") + + +def test_invalid_numexpr_version(): + for engine, parser in ENGINES_PARSERS: + yield check_invalid_numexpr_version, engine, parser + + +def check_invalid_numexpr_version(engine, parser): + def testit(): + a, b = 1, 2 + res = pd.eval('a + b', engine=engine, parser=parser) + tm.assert_equal(res, 3) + + if engine == 'numexpr': + try: + import numexpr as ne + except ImportError: + raise nose.SkipTest("no numexpr") + else: + if ne.__version__ < LooseVersion('2.1'): + with tm.assertRaisesRegexp(ImportError, "'numexpr' version is " + ".+, must be >= 2.1"): + testit() + elif ne.__version__ == LooseVersion('2.4.4'): + raise nose.SkipTest("numexpr version==2.4.4") + else: + testit() + else: + testit() + + +if __name__ == '__main__': + nose.runmodule(argv=[__file__, '-vvs', '-x', '--pdb', '--pdb-failure'], + exit=False) diff --git a/pandas/computation/tests/test_eval.py b/pandas/computation/tests/test_eval.py index b70252ed9f35b..518e5dd999391 100644 --- a/pandas/computation/tests/test_eval.py +++ b/pandas/computation/tests/test_eval.py @@ -1782,33 +1782,6 @@ def test_name_error_exprs(): yield check_name_error_exprs, engine, parser -def check_invalid_numexpr_version(engine, parser): - def testit(): - a, b = 1, 2 - res = pd.eval('a + b', engine=engine, parser=parser) - tm.assert_equal(res, 3) - - if engine == 'numexpr': - try: - import numexpr as ne - except ImportError: - raise nose.SkipTest("no numexpr") - else: - if ne.__version__ < LooseVersion('2.1'): - with tm.assertRaisesRegexp(ImportError, "'numexpr' version is " - ".+, must be >= 2.1"): - testit() - else: - testit() - else: - testit() - - -def test_invalid_numexpr_version(): - for engine, parser in ENGINES_PARSERS: - yield check_invalid_numexpr_version, engine, parser - - def check_invalid_local_variable_reference(engine, parser): tm.skip_if_no_ne(engine) From bb5cb1cbd23735bab9cbeaa466563676ba9f2ae0 Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Wed, 2 Mar 2016 11:26:25 -0500 Subject: [PATCH 2/2] COMPAT: remove boto dep check, xref #11915 --- ci/requirements-3.5.run | 1 + doc/source/whatsnew/v0.18.0.txt | 2 +- pandas/computation/expressions.py | 3 +++ pandas/core/datetools.py | 8 +++++--- pandas/core/format.py | 2 +- pandas/io/common.py | 4 ++++ pandas/util/clipboard.py | 1 + pandas/util/print_versions.py | 3 ++- 8 files changed, 18 insertions(+), 6 deletions(-) diff --git a/ci/requirements-3.5.run b/ci/requirements-3.5.run index 4ba3b473b3edd..fdc5f3f7dc992 100644 --- a/ci/requirements-3.5.run +++ b/ci/requirements-3.5.run @@ -18,6 +18,7 @@ sqlalchemy pymysql psycopg2 xarray +boto # incompat with conda ATM # beautiful-soup diff --git a/doc/source/whatsnew/v0.18.0.txt b/doc/source/whatsnew/v0.18.0.txt index 049152dba9a30..8d36d323f48f9 100644 --- a/doc/source/whatsnew/v0.18.0.txt +++ b/doc/source/whatsnew/v0.18.0.txt @@ -1125,7 +1125,7 @@ Bug Fixes - Bug in ``Series.resample`` using a frequency of ``Nano`` when the index is a ``DatetimeIndex`` and contains non-zero nanosecond parts (:issue:`12037`) - Bug in resampling with ``.nunique`` and a sparse index (:issue:`12352`) - Removed some compiler warnings (:issue:`12471`) - +- Work around compat issues with ``boto`` in python 3.5 (:issue:`11915`) - Bug in ``NaT`` subtraction from ``Timestamp`` or ``DatetimeIndex`` with timezones (:issue:`11718`) - Bug in subtraction of ``Series`` of a single tz-aware ``Timestamp`` (:issue:`12290`) diff --git a/pandas/computation/expressions.py b/pandas/computation/expressions.py index 82fbefd5ab608..086e92dbde1a0 100644 --- a/pandas/computation/expressions.py +++ b/pandas/computation/expressions.py @@ -11,6 +11,9 @@ from pandas.core.common import _values_from_object from pandas.computation import _NUMEXPR_INSTALLED +if _NUMEXPR_INSTALLED: + import numexpr as ne + _TEST_MODE = None _TEST_RESULT = None _USE_NUMEXPR = _NUMEXPR_INSTALLED diff --git a/pandas/core/datetools.py b/pandas/core/datetools.py index 91b33d30004b6..79718c79f9bdd 100644 --- a/pandas/core/datetools.py +++ b/pandas/core/datetools.py @@ -1,8 +1,10 @@ """A collection of random tools for dealing with dates in Python""" -from pandas.tseries.tools import * # noqa -from pandas.tseries.offsets import * # noqa -from pandas.tseries.frequencies import * # noqa +# flake8: noqa + +from pandas.tseries.tools import * +from pandas.tseries.offsets import * +from pandas.tseries.frequencies import * day = DateOffset() bday = BDay() diff --git a/pandas/core/format.py b/pandas/core/format.py index 101a5e64b65b5..1f1ff73869f73 100644 --- a/pandas/core/format.py +++ b/pandas/core/format.py @@ -10,7 +10,7 @@ from pandas.core.index import Index, MultiIndex, _ensure_index from pandas import compat from pandas.compat import (StringIO, lzip, range, map, zip, reduce, u, - OrderedDict) + OrderedDict, unichr) from pandas.util.terminal import get_terminal_size from pandas.core.config import get_option, set_option from pandas.io.common import _get_handle, UnicodeWriter, _expand_user diff --git a/pandas/io/common.py b/pandas/io/common.py index 8c9c348b9a11c..be8c3ccfe08e6 100644 --- a/pandas/io/common.py +++ b/pandas/io/common.py @@ -146,6 +146,10 @@ def readline(self): except ImportError: # boto is only needed for reading from S3. pass +except TypeError: + # boto/boto3 issues + # GH11915 + pass def _is_url(url): diff --git a/pandas/util/clipboard.py b/pandas/util/clipboard.py index 026f13aad0bf3..02da0d5b8159f 100644 --- a/pandas/util/clipboard.py +++ b/pandas/util/clipboard.py @@ -45,6 +45,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# flake8: noqa import platform import os diff --git a/pandas/util/print_versions.py b/pandas/util/print_versions.py index 80c10b53d37b5..c972caad5d74c 100644 --- a/pandas/util/print_versions.py +++ b/pandas/util/print_versions.py @@ -91,7 +91,8 @@ def show_versions(as_json=False): ("sqlalchemy", lambda mod: mod.__version__), ("pymysql", lambda mod: mod.__version__), ("psycopg2", lambda mod: mod.__version__), - ("jinja2", lambda mod: mod.__version__) + ("jinja2", lambda mod: mod.__version__), + ("boto", lambda mod: mod.__version__) ] deps_blob = list()