COMPAT: blacklist numexpr=2.4.4

closes #12489
closes #12511

Author: jreback

ci/requirements-3.4_SLOW.run

@@ -9,7 +9,7 @@ html5lib
 patsy
 beautiful-soup
 scipy
-numexpr
+numexpr=2.4.4
 pytables
 matplotlib
 lxml

doc/source/install.rst

@@ -225,7 +225,7 @@ Recommended Dependencies
 
 * `numexpr <https://github.com/pydata/numexpr>`__: for accelerating certain numerical operations.
   ``numexpr`` uses multiple cores as well as smart chunking and caching to achieve large speedups.
-  If installed, must be Version 2.1 or higher. Version 2.4.6 or higher on Windows is highly recommended.
+  If installed, must be Version 2.1 or higher (excluding a buggy 2.4.4). Version 2.4.6 or higher is highly recommended.
 
 * `bottleneck <http://berkeleyanalytics.com/bottleneck>`__: for accelerating certain types of ``nan``
   evaluations. ``bottleneck`` uses specialized cython routines to achieve large speedups.

doc/source/whatsnew/v0.18.0.txt

@@ -12,6 +12,10 @@ users upgrade to this version.
    pandas >= 0.18.0 no longer supports compatibility with Python version 2.6
    and 3.3 (:issue:`7718`, :issue:`11273`)
 
+.. warning::
+
+   ``numexpr`` version 2.4.4 will now show a warning and not be used as a computation back-end for pandas because of some buggy behavior. This does not affect other versions (>= 2.1 and >= 2.4.6). (:issue:`12489`)
+
 Highlights include:
 
 - Moving and expanding window functions are now methods on Series and DataFrame,

pandas/computation/__init__.py

@@ -0,0 +1,30 @@
+
+import warnings
+from distutils.version import LooseVersion
+
+_NUMEXPR_INSTALLED = False
+
+try:
+    import numexpr as ne
+    ver = ne.__version__
+    _NUMEXPR_INSTALLED = ver >= LooseVersion('2.1')
+
+    # we specifically disallow 2.4.4 as
+    # has some hard-to-diagnose bugs
+    if ver == LooseVersion('2.4.4'):
+        _NUMEXPR_INSTALLED = False
+        warnings.warn(
+            "The installed version of numexpr {ver} is not supported "
+            "in pandas and will be not be used\n".format(ver=ver),
+            UserWarning)
+
+    elif not _NUMEXPR_INSTALLED:
+        warnings.warn(
+            "The installed version of numexpr {ver} is not supported "
+            "in pandas and will be not be used\nThe minimum supported "
+            "version is 2.1\n".format(ver=ver), UserWarning)
+
+except ImportError:  # pragma: no cover
+    pass
+
+__all__ = ['_NUMEXPR_INSTALLED']

pandas/computation/eval.py

@@ -6,11 +6,11 @@
 import warnings
 import tokenize
 from pandas.core import common as com
+from pandas.computation import _NUMEXPR_INSTALLED
 from pandas.computation.expr import Expr, _parsers, tokenize_string
 from pandas.computation.scope import _ensure_scope
 from pandas.compat import string_types
 from pandas.computation.engines import _engines
-from distutils.version import LooseVersion
 
 
 def _check_engine(engine):
@@ -35,17 +35,11 @@ def _check_engine(engine):
     # that won't necessarily be import-able)
     # Could potentially be done on engine instantiation
     if engine == 'numexpr':
-        try:
-            import numexpr
-        except ImportError:
-            raise ImportError("'numexpr' not found. Cannot use "
+        if not _NUMEXPR_INSTALLED:
+            raise ImportError("'numexpr' is not installed or an "
+                              "unsupported version. Cannot use "
                               "engine='numexpr' for query/eval "
                               "if 'numexpr' is not installed")
-        else:
-            ne_version = numexpr.__version__
-            if ne_version < LooseVersion('2.1'):
-                raise ImportError("'numexpr' version is %s, "
-                                  "must be >= 2.1" % ne_version)
 
 
 def _check_parser(parser):

pandas/computation/expressions.py

@@ -9,20 +9,7 @@
 import warnings
 import numpy as np
 from pandas.core.common import _values_from_object
-from distutils.version import LooseVersion
-
-try:
-    import numexpr as ne
-    ver = ne.__version__
-    _NUMEXPR_INSTALLED = ver >= LooseVersion('2.1')
-    if not _NUMEXPR_INSTALLED:
-        warnings.warn(
-            "The installed version of numexpr {ver} is not supported "
-            "in pandas and will be not be used\nThe minimum supported "
-            "version is 2.1\n".format(ver=ver), UserWarning)
-
-except ImportError:  # pragma: no cover
-    _NUMEXPR_INSTALLED = False
+from pandas.computation import _NUMEXPR_INSTALLED
 
 _TEST_MODE = None
 _TEST_RESULT = None

pandas/computation/tests/test_compat.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+
+# flake8: noqa
+
+import nose
+from itertools import product
+from distutils.version import LooseVersion
+
+import pandas as pd
+from pandas.util import testing as tm
+
+from pandas.computation.engines import _engines
+import pandas.computation.expr as expr
+
+ENGINES_PARSERS = list(product(_engines, expr._parsers))
+
+
+def test_compat():
+    # test we have compat with our version of nu
+
+    from pandas.computation import _NUMEXPR_INSTALLED
+    try:
+        import numexpr as ne
+        ver = ne.__version__
+        if ver == LooseVersion('2.4.4'):
+            assert not _NUMEXPR_INSTALLED
+        elif ver < LooseVersion('2.1'):
+            with tm.assert_produces_warning(UserWarning,
+                                            check_stacklevel=False):
+                assert not _NUMEXPR_INSTALLED
+        else:
+            assert _NUMEXPR_INSTALLED
+
+    except ImportError:
+        raise nose.SkipTest("not testing numexpr version compat")
+
+
+def test_invalid_numexpr_version():
+    for engine, parser in ENGINES_PARSERS:
+        yield check_invalid_numexpr_version, engine, parser
+
+
+def check_invalid_numexpr_version(engine, parser):
+    def testit():
+        a, b = 1, 2
+        res = pd.eval('a + b', engine=engine, parser=parser)
+        tm.assert_equal(res, 3)
+
+    if engine == 'numexpr':
+        try:
+            import numexpr as ne
+        except ImportError:
+            raise nose.SkipTest("no numexpr")
+        else:
+            if ne.__version__ < LooseVersion('2.1'):
+                with tm.assertRaisesRegexp(ImportError, "'numexpr' version is "
+                                           ".+, must be >= 2.1"):
+                    testit()
+            elif ne.__version__ == LooseVersion('2.4.4'):
+                raise nose.SkipTest("numexpr version==2.4.4")
+            else:
+                testit()
+    else:
+        testit()
+
+
+if __name__ == '__main__':
+    nose.runmodule(argv=[__file__, '-vvs', '-x', '--pdb', '--pdb-failure'],
+                   exit=False)

pandas/computation/tests/test_eval.py

@@ -1782,33 +1782,6 @@ def test_name_error_exprs():
         yield check_name_error_exprs, engine, parser
 
 
-def check_invalid_numexpr_version(engine, parser):
-    def testit():
-        a, b = 1, 2
-        res = pd.eval('a + b', engine=engine, parser=parser)
-        tm.assert_equal(res, 3)
-
-    if engine == 'numexpr':
-        try:
-            import numexpr as ne
-        except ImportError:
-            raise nose.SkipTest("no numexpr")
-        else:
-            if ne.__version__ < LooseVersion('2.1'):
-                with tm.assertRaisesRegexp(ImportError, "'numexpr' version is "
-                                           ".+, must be >= 2.1"):
-                    testit()
-            else:
-                testit()
-    else:
-        testit()
-
-
-def test_invalid_numexpr_version():
-    for engine, parser in ENGINES_PARSERS:
-        yield check_invalid_numexpr_version, engine, parser
-
-
 def check_invalid_local_variable_reference(engine, parser):
     tm.skip_if_no_ne(engine)