COMPAT: blacklist numexpr=2.4.4

Closes #12489

Author: Jeff Reback <[email protected]>

Closes #12511 from jreback/numexpr and squashes the following commits:

bb5cb1c [Jeff Reback] COMPAT: remove boto dep check, xref #11915
bad7c0f [Jeff Reback] COMPAT: blacklist numexpr=2.4.4

Author: jreback

ci/requirements-3.4_SLOW.run

@@ -9,7 +9,7 @@ html5lib
 patsy
 beautiful-soup
 scipy
-numexpr
+numexpr=2.4.4
 pytables
 matplotlib
 lxml

ci/requirements-3.5.run

@@ -18,6 +18,7 @@ sqlalchemy
 pymysql
 psycopg2
 xarray
+boto
 
 # incompat with conda ATM
 # beautiful-soup

doc/source/install.rst

@@ -225,7 +225,7 @@ Recommended Dependencies
 
 * `numexpr <https://github.com/pydata/numexpr>`__: for accelerating certain numerical operations.
   ``numexpr`` uses multiple cores as well as smart chunking and caching to achieve large speedups.
-  If installed, must be Version 2.1 or higher. Version 2.4.6 or higher on Windows is highly recommended.
+  If installed, must be Version 2.1 or higher (excluding a buggy 2.4.4). Version 2.4.6 or higher is highly recommended.
 
 * `bottleneck <http://berkeleyanalytics.com/bottleneck>`__: for accelerating certain types of ``nan``
   evaluations. ``bottleneck`` uses specialized cython routines to achieve large speedups.

doc/source/whatsnew/v0.18.0.txt

@@ -12,6 +12,10 @@ users upgrade to this version.
    pandas >= 0.18.0 no longer supports compatibility with Python version 2.6
    and 3.3 (:issue:`7718`, :issue:`11273`)
 
+.. warning::
+
+   ``numexpr`` version 2.4.4 will now show a warning and not be used as a computation back-end for pandas because of some buggy behavior. This does not affect other versions (>= 2.1 and >= 2.4.6). (:issue:`12489`)
+
 Highlights include:
 
 - Moving and expanding window functions are now methods on Series and DataFrame,
@@ -1121,7 +1125,7 @@ Bug Fixes
 - Bug in ``Series.resample`` using a frequency of ``Nano`` when the index is a ``DatetimeIndex`` and contains non-zero nanosecond parts (:issue:`12037`)
 - Bug in resampling with ``.nunique`` and a sparse index (:issue:`12352`)
 - Removed some compiler warnings (:issue:`12471`)
-
+- Work around compat issues with ``boto`` in python 3.5 (:issue:`11915`)
 - Bug in ``NaT`` subtraction from ``Timestamp`` or ``DatetimeIndex`` with timezones (:issue:`11718`)
 - Bug in subtraction of ``Series`` of a single tz-aware ``Timestamp`` (:issue:`12290`)
 

pandas/computation/__init__.py

@@ -0,0 +1,30 @@
+
+import warnings
+from distutils.version import LooseVersion
+
+_NUMEXPR_INSTALLED = False
+
+try:
+    import numexpr as ne
+    ver = ne.__version__
+    _NUMEXPR_INSTALLED = ver >= LooseVersion('2.1')
+
+    # we specifically disallow 2.4.4 as
+    # has some hard-to-diagnose bugs
+    if ver == LooseVersion('2.4.4'):
+        _NUMEXPR_INSTALLED = False
+        warnings.warn(
+            "The installed version of numexpr {ver} is not supported "
+            "in pandas and will be not be used\n".format(ver=ver),
+            UserWarning)
+
+    elif not _NUMEXPR_INSTALLED:
+        warnings.warn(
+            "The installed version of numexpr {ver} is not supported "
+            "in pandas and will be not be used\nThe minimum supported "
+            "version is 2.1\n".format(ver=ver), UserWarning)
+
+except ImportError:  # pragma: no cover
+    pass
+
+__all__ = ['_NUMEXPR_INSTALLED']

pandas/computation/eval.py

@@ -6,11 +6,11 @@
 import warnings
 import tokenize
 from pandas.core import common as com
+from pandas.computation import _NUMEXPR_INSTALLED
 from pandas.computation.expr import Expr, _parsers, tokenize_string
 from pandas.computation.scope import _ensure_scope
 from pandas.compat import string_types
 from pandas.computation.engines import _engines
-from distutils.version import LooseVersion
 
 
 def _check_engine(engine):
@@ -35,17 +35,11 @@ def _check_engine(engine):
     # that won't necessarily be import-able)
     # Could potentially be done on engine instantiation
     if engine == 'numexpr':
-        try:
-            import numexpr
-        except ImportError:
-            raise ImportError("'numexpr' not found. Cannot use "
+        if not _NUMEXPR_INSTALLED:
+            raise ImportError("'numexpr' is not installed or an "
+                              "unsupported version. Cannot use "
                               "engine='numexpr' for query/eval "
                               "if 'numexpr' is not installed")
-        else:
-            ne_version = numexpr.__version__
-            if ne_version < LooseVersion('2.1'):
-                raise ImportError("'numexpr' version is %s, "
-                                  "must be >= 2.1" % ne_version)
 
 
 def _check_parser(parser):

pandas/computation/expressions.py

@@ -9,20 +9,10 @@
 import warnings
 import numpy as np
 from pandas.core.common import _values_from_object
-from distutils.version import LooseVersion
+from pandas.computation import _NUMEXPR_INSTALLED
 
-try:
+if _NUMEXPR_INSTALLED:
     import numexpr as ne
-    ver = ne.__version__
-    _NUMEXPR_INSTALLED = ver >= LooseVersion('2.1')
-    if not _NUMEXPR_INSTALLED:
-        warnings.warn(
-            "The installed version of numexpr {ver} is not supported "
-            "in pandas and will be not be used\nThe minimum supported "
-            "version is 2.1\n".format(ver=ver), UserWarning)
-
-except ImportError:  # pragma: no cover
-    _NUMEXPR_INSTALLED = False
 
 _TEST_MODE = None
 _TEST_RESULT = None

pandas/computation/tests/test_compat.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+
+# flake8: noqa
+
+import nose
+from itertools import product
+from distutils.version import LooseVersion
+
+import pandas as pd
+from pandas.util import testing as tm
+
+from pandas.computation.engines import _engines
+import pandas.computation.expr as expr
+
+ENGINES_PARSERS = list(product(_engines, expr._parsers))
+
+
+def test_compat():
+    # test we have compat with our version of nu
+
+    from pandas.computation import _NUMEXPR_INSTALLED
+    try:
+        import numexpr as ne
+        ver = ne.__version__
+        if ver == LooseVersion('2.4.4'):
+            assert not _NUMEXPR_INSTALLED
+        elif ver < LooseVersion('2.1'):
+            with tm.assert_produces_warning(UserWarning,
+                                            check_stacklevel=False):
+                assert not _NUMEXPR_INSTALLED
+        else:
+            assert _NUMEXPR_INSTALLED
+
+    except ImportError:
+        raise nose.SkipTest("not testing numexpr version compat")
+
+
+def test_invalid_numexpr_version():
+    for engine, parser in ENGINES_PARSERS:
+        yield check_invalid_numexpr_version, engine, parser
+
+
+def check_invalid_numexpr_version(engine, parser):
+    def testit():
+        a, b = 1, 2
+        res = pd.eval('a + b', engine=engine, parser=parser)
+        tm.assert_equal(res, 3)
+
+    if engine == 'numexpr':
+        try:
+            import numexpr as ne
+        except ImportError:
+            raise nose.SkipTest("no numexpr")
+        else:
+            if ne.__version__ < LooseVersion('2.1'):
+                with tm.assertRaisesRegexp(ImportError, "'numexpr' version is "
+                                           ".+, must be >= 2.1"):
+                    testit()
+            elif ne.__version__ == LooseVersion('2.4.4'):
+                raise nose.SkipTest("numexpr version==2.4.4")
+            else:
+                testit()
+    else:
+        testit()
+
+
+if __name__ == '__main__':
+    nose.runmodule(argv=[__file__, '-vvs', '-x', '--pdb', '--pdb-failure'],
+                   exit=False)

pandas/computation/tests/test_eval.py

@@ -1782,33 +1782,6 @@ def test_name_error_exprs():
         yield check_name_error_exprs, engine, parser
 
 
-def check_invalid_numexpr_version(engine, parser):
-    def testit():
-        a, b = 1, 2
-        res = pd.eval('a + b', engine=engine, parser=parser)
-        tm.assert_equal(res, 3)
-
-    if engine == 'numexpr':
-        try:
-            import numexpr as ne
-        except ImportError:
-            raise nose.SkipTest("no numexpr")
-        else:
-            if ne.__version__ < LooseVersion('2.1'):
-                with tm.assertRaisesRegexp(ImportError, "'numexpr' version is "
-                                           ".+, must be >= 2.1"):
-                    testit()
-            else:
-                testit()
-    else:
-        testit()
-
-
-def test_invalid_numexpr_version():
-    for engine, parser in ENGINES_PARSERS:
-        yield check_invalid_numexpr_version, engine, parser
-
-
 def check_invalid_local_variable_reference(engine, parser):
     tm.skip_if_no_ne(engine)
 

pandas/core/datetools.py

@@ -1,8 +1,10 @@
 """A collection of random tools for dealing with dates in Python"""
 
-from pandas.tseries.tools import *  # noqa
-from pandas.tseries.offsets import *  # noqa
-from pandas.tseries.frequencies import *  # noqa
+# flake8: noqa
+
+from pandas.tseries.tools import *
+from pandas.tseries.offsets import *
+from pandas.tseries.frequencies import *
 
 day = DateOffset()
 bday = BDay()

pandas/core/format.py

@@ -10,7 +10,7 @@
 from pandas.core.index import Index, MultiIndex, _ensure_index
 from pandas import compat
 from pandas.compat import (StringIO, lzip, range, map, zip, reduce, u,
-                           OrderedDict)
+                           OrderedDict, unichr)
 from pandas.util.terminal import get_terminal_size
 from pandas.core.config import get_option, set_option
 from pandas.io.common import _get_handle, UnicodeWriter, _expand_user

pandas/io/common.py

@@ -146,6 +146,10 @@ def readline(self):
 except ImportError:
     # boto is only needed for reading from S3.
     pass
+except TypeError:
+    # boto/boto3 issues
+    # GH11915
+    pass
 
 
 def _is_url(url):

pandas/util/clipboard.py

@@ -45,6 +45,7 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+# flake8: noqa
 
 import platform
 import os

pandas/util/print_versions.py

@@ -91,7 +91,8 @@ def show_versions(as_json=False):
         ("sqlalchemy", lambda mod: mod.__version__),
         ("pymysql", lambda mod: mod.__version__),
         ("psycopg2", lambda mod: mod.__version__),
-        ("jinja2", lambda mod: mod.__version__)
+        ("jinja2", lambda mod: mod.__version__),
+        ("boto", lambda mod: mod.__version__)
     ]
 
     deps_blob = list()