TypeError when subtracting datetime64 and timestamp but only in eval

[alexmojaki]

Code Sample

import pandas as pd
pd.show_versions()

today = pd.to_datetime("today")
df = pd.DataFrame({'date': [today]})

assert isinstance(today, pd.Timestamp)
assert str(df.date.dtype) == 'datetime64[ns]'

delta = df.date - today  # works fine
assert str(delta.dtype) == 'timedelta64[ns]'

df.eval("date - @today")  # fails

Live demo: https://repl.it/repls/SelfassuredFrighteningNumber

Problem description

Subtraction works 'normally' but not when used inside DataFrame.eval or .query. AFAIK the two methods should be equivalent. eval fails with:

Traceback (most recent call last):
  File "main.py", line 13, in <module>
    df.eval("date - @today")  # fails
  File "/home/runner/.local/share/virtualenvs/python3/lib/python3.7/site-packages/pandas/core/frame.py", line 3315, in eval
    return _eval(expr, inplace=inplace, **kwargs)
  File "/home/runner/.local/share/virtualenvs/python3/lib/python3.7/site-packages/pandas/core/computation/eval.py", line 322, in eval
    parsed_expr = Expr(expr, engine=engine, parser=parser, env=env, truediv=truediv)
  File "/home/runner/.local/share/virtualenvs/python3/lib/python3.7/site-packages/pandas/core/computation/expr.py", line 830, in __init__
    self.terms = self.parse()
  File "/home/runner/.local/share/virtualenvs/python3/lib/python3.7/site-packages/pandas/core/computation/expr.py", line 847, in parse
    return self._visitor.visit(self.expr)
  File "/home/runner/.local/share/virtualenvs/python3/lib/python3.7/site-packages/pandas/core/computation/expr.py", line 441, in visit
    return visitor(node, **kwargs)
  File "/home/runner/.local/share/virtualenvs/python3/lib/python3.7/site-packages/pandas/core/computation/expr.py", line 447, in visit_Module
    return self.visit(expr, **kwargs)
  File "/home/runner/.local/share/virtualenvs/python3/lib/python3.7/site-packages/pandas/core/computation/expr.py", line 441, in visit
    return visitor(node, **kwargs)
  File "/home/runner/.local/share/virtualenvs/python3/lib/python3.7/site-packages/pandas/core/computation/expr.py", line 450, in visit_Expr
    return self.visit(node.value, **kwargs)
  File "/home/runner/.local/share/virtualenvs/python3/lib/python3.7/site-packages/pandas/core/computation/expr.py", line 441, in visit
    return visitor(node, **kwargs)
  File "/home/runner/.local/share/virtualenvs/python3/lib/python3.7/site-packages/pandas/core/computation/expr.py", line 565, in visit_BinOp
    return self._maybe_evaluate_binop(op, op_class, left, right)
  File "/home/runner/.local/share/virtualenvs/python3/lib/python3.7/site-packages/pandas/core/computation/expr.py", line 536, in _maybe_evaluate_binop
    " '{lhs}' and '{rhs}'".format(op=res.op, lhs=lhs.type, rhs=rhs.type)
TypeError: unsupported operand type(s) for -: 'datetime64[ns]' and '<class 'pandas._libs.tslibs.timestamps.Timestamp'>'

Output of pd.show_versions()

INSTALLED VERSIONS

commit : None
python : 3.7.4.final.0
python-bits : 64
OS : Linux
OS-release : 4.15.0-1036-gcp
machine : x86_64
processor :
byteorder : little
LC_ALL : None
LANG : C.UTF-8
LOCALE : en_US.UTF-8

pandas : 0.25.3numpy : 1.18.0
pytz : 2019.3
dateutil : 2.8.1pip : 19.0.3
setuptools : 40.8.0
Cython : Nonepytest : None
hypothesis : None
sphinx : Noneblosc : None
feather : None
xlsxwriter : Nonelxml.etree : None
html5lib : None
pymysql : Nonepsycopg2 : None
jinja2 : None
IPython : Nonepandas_datareader: None
bs4 : None
bottleneck : Nonefastparquet : None
gcsfs : None
lxml.etree : None
matplotlib : 3.1.1
numexpr : None
odfpy : None
openpyxl : None
pandas_gbq : None
pyarrow : None
pytables : None
s3fs : None
scipy : 1.3.1
sqlalchemy : None
tables : None
xarray : None
xlrd : None
xlwt : None
xlsxwriter : None

[alexmojaki]

Perhaps related issue I found: #28286

[jreback]

pls try on master

[alexmojaki]

I don't have that set up personally, can you just copy paste the script?

[alexmojaki]

I just tried this on master and got the same thing:

INSTALLED VERSIONS
------------------
commit           : 4018550c796f9b23369267565b8ab8c87b9d14ad
python           : 3.8.0.final.0
python-bits      : 64
OS               : Linux
OS-release       : 5.0.0-37-generic
Version          : #40~18.04.1-Ubuntu SMP Thu Nov 14 12:06:39 UTC 2019
machine          : x86_64
processor        : x86_64
byteorder        : little
LC_ALL           : None
LANG             : en_ZA.UTF-8
LOCALE           : en_ZA.UTF-8

pandas           : 1.1.0.dev0+673.g4018550c7
numpy            : 1.18.1
pytz             : 2019.3
dateutil         : 2.8.1
pip              : 19.0.3
setuptools       : 40.8.0
Cython           : None
pytest           : 5.3.5
hypothesis       : 5.6.0
sphinx           : None
blosc            : None
feather          : None
xlsxwriter       : None
lxml.etree       : None
html5lib         : None
pymysql          : None
psycopg2         : None
jinja2           : None
IPython          : None
pandas_datareader: None
bs4              : None
bottleneck       : None
fastparquet      : None
gcsfs            : None
matplotlib       : None
numexpr          : None
odfpy            : None
openpyxl         : None
pandas_gbq       : None
pyarrow          : None
pytables         : None
pyxlsb           : None
s3fs             : None
scipy            : None
sqlalchemy       : None
tables           : None
tabulate         : None
xarray           : None
xlrd             : None
xlwt             : None
numba            : None
Traceback (most recent call last):
  File "/home/alex/.PyCharm2019.3/config/scratches/scratch_576.py", line 13, in <module>
    df.eval("date - @today")  # fails
  File "/home/alex/work/pandas/pandas/core/frame.py", line 3078, in eval
    return _eval(expr, inplace=inplace, **kwargs)
  File "/home/alex/work/pandas/pandas/core/computation/eval.py", line 342, in eval
    parsed_expr = Expr(expr, engine=engine, parser=parser, env=env)
  File "/home/alex/work/pandas/pandas/core/computation/expr.py", line 786, in __init__
    self.terms = self.parse()
  File "/home/alex/work/pandas/pandas/core/computation/expr.py", line 805, in parse
    return self._visitor.visit(self.expr)
  File "/home/alex/work/pandas/pandas/core/computation/expr.py", line 398, in visit
    return visitor(node, **kwargs)
  File "/home/alex/work/pandas/pandas/core/computation/expr.py", line 404, in visit_Module
    return self.visit(expr, **kwargs)
  File "/home/alex/work/pandas/pandas/core/computation/expr.py", line 398, in visit
    return visitor(node, **kwargs)
  File "/home/alex/work/pandas/pandas/core/computation/expr.py", line 407, in visit_Expr
    return self.visit(node.value, **kwargs)
  File "/home/alex/work/pandas/pandas/core/computation/expr.py", line 398, in visit
    return visitor(node, **kwargs)
  File "/home/alex/work/pandas/pandas/core/computation/expr.py", line 522, in visit_BinOp
    return self._maybe_evaluate_binop(op, op_class, left, right)
  File "/home/alex/work/pandas/pandas/core/computation/expr.py", line 491, in _maybe_evaluate_binop
    raise TypeError(
TypeError: unsupported operand type(s) for -: 'datetime64[ns]' and '<class 'pandas._libs.tslibs.timestamps.Timestamp'>'