Int64 can't be used in pandas.eval()

[antonov-andrey]

This code works fine

import pandas as pd

d = pd.DataFrame([[1, 2], [3, 4]], columns=['a', 'b'], dtype="Float64")
d.eval('c = b - a', inplace=True)

This code fails with the following stacktrace below

d = pd.DataFrame([[1, 2], [3, 4]], columns=['a', 'b'], dtype="Int64")
d.eval('c = b - a', inplace=True)

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-43-42645933ba81> in <module>
----> 1 d.eval('c = b - a', inplace=True)
      2 d

~/.local/lib/python3.6/site-packages/pandas/core/frame.py in eval(self, expr, inplace, **kwargs)
   3313             kwargs["target"] = self
   3314         kwargs["resolvers"] = kwargs.get("resolvers", ()) + tuple(resolvers)
-> 3315         return _eval(expr, inplace=inplace, **kwargs)
   3316 
   3317     def select_dtypes(self, include=None, exclude=None):

~/.local/lib/python3.6/site-packages/pandas/core/computation/eval.py in eval(expr, parser, engine, truediv, local_dict, global_dict, resolvers, level, target, inplace)
    320         )
    321 
--> 322         parsed_expr = Expr(expr, engine=engine, parser=parser, env=env, truediv=truediv)
    323 
    324         # construct the engine and evaluate the parsed expression

~/.local/lib/python3.6/site-packages/pandas/core/computation/expr.py in __init__(self, expr, engine, parser, env, truediv, level)
    828         self.env.scope["truediv"] = truediv
    829         self._visitor = _parsers[parser](self.env, self.engine, self.parser)
--> 830         self.terms = self.parse()
    831 
    832     @property

~/.local/lib/python3.6/site-packages/pandas/core/computation/expr.py in parse(self)
    845     def parse(self):
    846         """Parse an expression"""
--> 847         return self._visitor.visit(self.expr)
    848 
    849     @property

~/.local/lib/python3.6/site-packages/pandas/core/computation/expr.py in visit(self, node, **kwargs)
    439         method = "visit_" + node.__class__.__name__
    440         visitor = getattr(self, method)
--> 441         return visitor(node, **kwargs)
    442 
    443     def visit_Module(self, node, **kwargs):

~/.local/lib/python3.6/site-packages/pandas/core/computation/expr.py in visit_Module(self, node, **kwargs)
    445             raise SyntaxError("only a single expression is allowed")
    446         expr = node.body[0]
--> 447         return self.visit(expr, **kwargs)
    448 
    449     def visit_Expr(self, node, **kwargs):

~/.local/lib/python3.6/site-packages/pandas/core/computation/expr.py in visit(self, node, **kwargs)
    439         method = "visit_" + node.__class__.__name__
    440         visitor = getattr(self, method)
--> 441         return visitor(node, **kwargs)
    442 
    443     def visit_Module(self, node, **kwargs):

~/.local/lib/python3.6/site-packages/pandas/core/computation/expr.py in visit_Assign(self, node, **kwargs)
    663             )
    664 
--> 665         return self.visit(node.value, **kwargs)
    666 
    667     def visit_Attribute(self, node, **kwargs):

~/.local/lib/python3.6/site-packages/pandas/core/computation/expr.py in visit(self, node, **kwargs)
    439         method = "visit_" + node.__class__.__name__
    440         visitor = getattr(self, method)
--> 441         return visitor(node, **kwargs)
    442 
    443     def visit_Module(self, node, **kwargs):

~/.local/lib/python3.6/site-packages/pandas/core/computation/expr.py in visit_BinOp(self, node, **kwargs)
    563         op, op_class, left, right = self._maybe_transform_eq_ne(node)
    564         left, right = self._maybe_downcast_constants(left, right)
--> 565         return self._maybe_evaluate_binop(op, op_class, left, right)
    566 
    567     def visit_Div(self, node, **kwargs):

~/.local/lib/python3.6/site-packages/pandas/core/computation/expr.py in _maybe_evaluate_binop(self, op, op_class, lhs, rhs, eval_in_python, maybe_eval_in_python)
    531         res = op(lhs, rhs)
    532 
--> 533         if res.has_invalid_return_type:
    534             raise TypeError(
    535                 "unsupported operand type(s) for {op}:"

~/.local/lib/python3.6/site-packages/pandas/core/computation/ops.py in has_invalid_return_type(self)
    223         types = self.operand_types
    224         obj_dtype_set = frozenset([np.dtype("object")])
--> 225         return self.return_type == object and types - obj_dtype_set
    226 
    227     @property

~/.local/lib/python3.6/site-packages/pandas/core/computation/ops.py in return_type(self)
    217         if self.op in (_cmp_ops_syms + _bool_ops_syms):
    218             return np.bool_
--> 219         return _result_type_many(*(term.type for term in com.flatten(self)))
    220 
    221     @property

~/.local/lib/python3.6/site-packages/pandas/core/computation/common.py in _result_type_many(*arrays_and_dtypes)
     20     argument limit """
     21     try:
---> 22         return np.result_type(*arrays_and_dtypes)
     23     except ValueError:
     24         # we have > NPY_MAXARGS terms in our expression

<__array_function__ internals> in result_type(*args, **kwargs)

TypeError: data type not understood

Output of pd.show_versions()

INSTALLED VERSIONS ------------------ commit : None python : 3.6.8.final.0 python-bits : 64 OS : Linux OS-release : 5.0.0-35-generic machine : x86_64 processor : x86_64 byteorder : little LC_ALL : None LANG : en_US.UTF-8 LOCALE : en_US.UTF-8

pandas : 0.25.3
numpy : 1.17.4
pytz : 2019.3
dateutil : 2.8.1
pip : 9.0.1
setuptools : 41.6.0
Cython : 0.29.14
pytest : None
hypothesis : None
sphinx : None
blosc : None
feather : None
xlsxwriter : None
lxml.etree : None
html5lib : 0.999999999
pymysql : None
psycopg2 : 2.8.4 (dt dec pq3 ext lo64)
jinja2 : 2.10.3
IPython : 7.9.0
pandas_datareader: None
bs4 : None
bottleneck : 1.3.0
fastparquet : 0.3.2
gcsfs : None
lxml.etree : None
matplotlib : 3.1.1
numexpr : 2.7.0
odfpy : None
openpyxl : None
pandas_gbq : None
pyarrow : 0.15.1
pytables : None
s3fs : 0.3.5
scipy : 1.3.2
sqlalchemy : None
tables : None
xarray : None
xlrd : None
xlwt : None
xlsxwriter : None

[jreback]

numexpr has 0 support for extension types and will likely never have. should just shift this to using the ```engine='python'`` for extension dtypes. a community PR would be needed here.

[antonov-andrey]

Thank you for the fast reply! But when I try to use it with engine='python':
d.eval('c = b - a', inplace=True, engine='python')
it do not work exactly the same way as I described.

[stefansimik]

Same problem here.
engine='python' does not help.

Float - works fine

(
    pd.DataFrame({'A': [1, 2, np.NaN], 'B': [1, np.NaN, 3]})
    .astype('float')
    .eval('C = A + B', engine='python')
)

Int64 - Fails:

(
    pd.DataFrame({'A': [1, 2, np.NaN], 'B': [1, np.NaN, 3]})
    .astype('Int64')                 #  <--- nullable integer causes fail in following eval(..)
    .eval('C = A + B', engine='python')
)

[evyasonov]

Any update?

[jreback]

@evyasonov you are welcome to contribute a patch