BUG/TST: run and fix all arithmetic tests with+without numexpr

pandas/_libs/tslibs/nattype.pyx

@@ -127,6 +127,10 @@ cdef class _NaT(datetime):
                 result.fill(_nat_scalar_rules[op])
             elif other.dtype.kind == "O":
                 result = np.array([PyObject_RichCompare(self, x, op) for x in other])
+            elif op == Py_EQ:
+                result = np.zeros(other.shape, dtype=bool)
+            elif op == Py_NE:
+                result = np.ones(other.shape, dtype=bool)
             else:
                 return NotImplemented
             return result

pandas/core/ops/array_ops.py

@@ -2,19 +2,21 @@
 Functions for arithmetic and comparison operations on NumPy arrays and
 ExtensionArrays.
 """
-from datetime import timedelta
+import datetime
 from functools import partial
 import operator
 from typing import Any
 
 import numpy as np
 
 from pandas._libs import (
+    NaTType,
     Timedelta,
     Timestamp,
     lib,
     ops as libops,
 )
+from pandas._libs.tslibs import BaseOffset
 from pandas._typing import (
     ArrayLike,
     Shape,
@@ -155,8 +157,14 @@ def _na_arithmetic_op(left, right, op, is_cmp: bool = False):
     ------
     TypeError : invalid operation
     """
+    if isinstance(right, str):
+        # can never use numexpr
+        func = op
+    else:
+        func = partial(expressions.evaluate, op)
+
     try:
-        result = expressions.evaluate(op, left, right)
+        result = func(left, right)
     except TypeError:
         if is_cmp:
             # numexpr failed on comparison op, e.g. ndarray[float] > datetime
@@ -202,7 +210,9 @@ def arithmetic_op(left: ArrayLike, right: Any, op):
     rvalues = ensure_wrapped_if_datetimelike(right)
     rvalues = _maybe_upcast_for_op(rvalues, lvalues.shape)
 
-    if should_extension_dispatch(lvalues, rvalues) or isinstance(rvalues, Timedelta):
+    if should_extension_dispatch(lvalues, rvalues) or isinstance(
+        rvalues, (Timedelta, BaseOffset, Timestamp, NaTType)
+    ):
         # Timedelta is included because numexpr will fail on it, see GH#31457
         res_values = op(lvalues, rvalues)
 
@@ -248,7 +258,9 @@ def comparison_op(left: ArrayLike, right: Any, op) -> ArrayLike:
                 "Lengths must match to compare", lvalues.shape, rvalues.shape
             )
 
-    if should_extension_dispatch(lvalues, rvalues):
+    if should_extension_dispatch(lvalues, rvalues) or isinstance(
+        rvalues, (Timedelta, BaseOffset, Timestamp, NaTType)
+    ):
         # Call the method on lvalues
         res_values = op(lvalues, rvalues)
 
@@ -263,7 +275,7 @@ def comparison_op(left: ArrayLike, right: Any, op) -> ArrayLike:
         # GH#36377 going through the numexpr path would incorrectly raise
         return invalid_comparison(lvalues, rvalues, op)
 
-    elif is_object_dtype(lvalues.dtype):
+    elif is_object_dtype(lvalues.dtype) or isinstance(rvalues, str):
         res_values = comp_method_OBJECT_ARRAY(op, lvalues, rvalues)
 
     else:
@@ -440,11 +452,13 @@ def _maybe_upcast_for_op(obj, shape: Shape):
     Be careful to call this *after* determining the `name` attribute to be
     attached to the result of the arithmetic operation.
     """
-    if type(obj) is timedelta:
+    if type(obj) is datetime.timedelta:
         # GH#22390  cast up to Timedelta to rely on Timedelta
         # implementation; otherwise operation against numeric-dtype
         # raises TypeError
         return Timedelta(obj)
+    elif type(obj) is datetime.datetime:
+        return Timestamp(obj)
     elif isinstance(obj, np.datetime64):
         # GH#28080 numpy casts integer-dtype to datetime64 when doing
         #  array[int] + datetime64, which we do not allow

pandas/tests/arithmetic/conftest.py

@@ -9,6 +9,18 @@
     UInt64Index,
 )
 import pandas._testing as tm
+from pandas.core.computation import expressions as expr
+
+
+@pytest.fixture(
+    autouse=True, scope="module", params=[0, 1000000], ids=["numexpr", "python"]
+)
+def switch_numexpr_min_elements(request):
+    _MIN_ELEMENTS = expr._MIN_ELEMENTS
+    expr._MIN_ELEMENTS = request.param
+    yield request.param
+    expr._MIN_ELEMENTS = _MIN_ELEMENTS
+
 
 # ------------------------------------------------------------------
 # Helper Functions

pandas/tests/arithmetic/test_numeric.py

@@ -27,6 +27,7 @@
 )
 import pandas._testing as tm
 from pandas.core import ops
+from pandas.core.computation import expressions as expr
 
 
 @pytest.fixture(params=[Index, Series, tm.to_array])
@@ -127,7 +128,12 @@ def test_numeric_cmp_string_numexpr_path(self, box_with_array):
         result = obj != "a"
         tm.assert_equal(result, ~expected)
 
-        msg = "Invalid comparison between dtype=float64 and str"
+        msg = "|".join(
+            [
+                "Invalid comparison between dtype=float64 and str",
+                "'<' not supported between instances of 'numpy.ndarray' and 'str'",
+            ]
+        )
         with pytest.raises(TypeError, match=msg):
             obj < "a"
 
@@ -390,7 +396,7 @@ def test_div_negative_zero(self, zero, numeric_idx, op):
     # ------------------------------------------------------------------
 
     @pytest.mark.parametrize("dtype1", [np.int64, np.float64, np.uint64])
-    def test_ser_div_ser(self, dtype1, any_real_dtype):
+    def test_ser_div_ser(self, switch_numexpr_min_elements, dtype1, any_real_dtype):
         # no longer do integer div for any ops, but deal with the 0's
         dtype2 = any_real_dtype
 
@@ -404,6 +410,11 @@ def test_ser_div_ser(self, dtype1, any_real_dtype):
                 name=None,
             )
         expected.iloc[0:3] = np.inf
+        if first.dtype == "int64" and second.dtype == "float32":
+            # when using numexpr, the casting rules are slightly different
+            # and int64/float32 combo results in float32 instead of float64
+            if expr.USE_NUMEXPR and switch_numexpr_min_elements == 0:
+                expected = expected.astype("float32")
 
         result = first / second
         tm.assert_series_equal(result, expected)

pandas/tests/frame/test_arithmetic.py

@@ -174,9 +174,19 @@ def test_timestamp_compare(self):
                 with pytest.raises(TypeError, match=msg):
                     right_f(pd.Timestamp("20010109"), df)
             # nats
-            expected = left_f(df, pd.Timestamp("nat"))
-            result = right_f(pd.Timestamp("nat"), df)
-            tm.assert_frame_equal(result, expected)
+            if left in ["eq", "ne"]:
+                expected = left_f(df, pd.Timestamp("nat"))
+                result = right_f(pd.Timestamp("nat"), df)
+                tm.assert_frame_equal(result, expected)
+            else:
+                msg = (
+                    "'(<|>)=?' not supported between "
+                    "instances of 'numpy.ndarray' and 'NaTType'"
+                )
+                with pytest.raises(TypeError, match=msg):
+                    left_f(df, pd.Timestamp("nat"))
+                with pytest.raises(TypeError, match=msg):
+                    right_f(pd.Timestamp("nat"), df)
 
     def test_mixed_comparison(self):
         # GH#13128, GH#22163 != datetime64 vs non-dt64 should be False,