Merge remote-tracking branch 'upstream/master' into feature/validate_docstrings

Author: FHaase

doc/source/whatsnew/v0.24.0.txt

@@ -1133,6 +1133,7 @@ Interval
 - Bug in the ``IntervalIndex`` repr where a trailing comma was missing after the list of intervals (:issue:`20611`)
 - Bug in :class:`Interval` where scalar arithmetic operations did not retain the ``closed`` value (:issue:`22313`)
 - Bug in :class:`IntervalIndex` where indexing with datetime-like values raised a ``KeyError`` (:issue:`20636`)
+- Bug in ``IntervalTree`` where data containing ``NaN`` triggered a warning and resulted in incorrect indexing queries with :class:`IntervalIndex` (:issue:`23352`)
 
 Indexing
 ^^^^^^^^

pandas/_libs/intervaltree.pxi.in

@@ -72,6 +72,12 @@ cdef class IntervalTree(IntervalMixin):
 
         self.closed = closed
 
+        # GH 23352: ensure no nan in nodes
+        mask = ~np.isnan(self.left)
+        self.left = self.left[mask]
+        self.right = self.right[mask]
+        indices = indices[mask]
+
         node_cls = NODE_CLASSES[str(self.dtype), closed]
         self.root = node_cls(self.left, self.right, indices, leaf_size)
 

pandas/core/frame.py

@@ -4944,7 +4944,7 @@ def _combine_match_columns(self, other, func, level=None):
         assert left.columns.equals(right.index)
         return ops.dispatch_to_series(left, right, func, axis="columns")
 
-    def _combine_const(self, other, func, errors='raise'):
+    def _combine_const(self, other, func):
         assert lib.is_scalar(other) or np.ndim(other) == 0
         return ops.dispatch_to_series(self, other, func)
 

pandas/core/indexes/base.py

@@ -4702,6 +4702,13 @@ def dropna(self, how='any'):
     def _evaluate_with_timedelta_like(self, other, op):
         # Timedelta knows how to operate with np.array, so dispatch to that
         # operation and then wrap the results
+        if self._is_numeric_dtype and op.__name__ in ['add', 'sub',
+                                                      'radd', 'rsub']:
+            raise TypeError("Operation {opname} between {cls} and {other} "
+                            "is invalid".format(opname=op.__name__,
+                                                cls=type(self).__name__,
+                                                other=type(other).__name__))
+
         other = Timedelta(other)
         values = self.values
 

pandas/core/indexes/range.py

@@ -14,7 +14,8 @@
 from pandas.core.dtypes import concat as _concat
 from pandas.core.dtypes.common import (
     is_int64_dtype, is_integer, is_scalar, is_timedelta64_dtype)
-from pandas.core.dtypes.generic import ABCSeries, ABCTimedeltaIndex
+from pandas.core.dtypes.generic import (
+    ABCDataFrame, ABCSeries, ABCTimedeltaIndex)
 
 from pandas.core import ops
 import pandas.core.common as com
@@ -558,6 +559,9 @@ def __getitem__(self, key):
         return super_getitem(key)
 
     def __floordiv__(self, other):
+        if isinstance(other, (ABCSeries, ABCDataFrame)):
+            return NotImplemented
+
         if is_integer(other) and other != 0:
             if (len(self) == 0 or
                     self._start % other == 0 and
@@ -589,7 +593,7 @@ def _make_evaluate_binop(op, step=False):
             """
 
             def _evaluate_numeric_binop(self, other):
-                if isinstance(other, ABCSeries):
+                if isinstance(other, (ABCSeries, ABCDataFrame)):
                     return NotImplemented
                 elif isinstance(other, ABCTimedeltaIndex):
                     # Defer to TimedeltaIndex implementation

pandas/core/ops.py

@@ -130,6 +130,13 @@ def maybe_upcast_for_op(obj):
         # implementation; otherwise operation against numeric-dtype
         # raises TypeError
         return pd.Timedelta(obj)
+    elif isinstance(obj, np.timedelta64) and not isna(obj):
+        # In particular non-nanosecond timedelta64 needs to be cast to
+        #  nanoseconds, or else we get undesired behavior like
+        #  np.timedelta64(3, 'D') / 2 == np.timedelta64(1, 'D')
+        # The isna check is to avoid casting timedelta64("NaT"), which would
+        #  return NaT and incorrectly be treated as a datetime-NaT.
+        return pd.Timedelta(obj)
     elif isinstance(obj, np.ndarray) and is_timedelta64_dtype(obj):
         # GH#22390 Unfortunately we need to special-case right-hand
         # timedelta64 dtypes because numpy casts integer dtypes to
@@ -1405,11 +1412,12 @@ def wrapper(left, right):
                                     index=left.index, name=res_name,
                                     dtype=result.dtype)
 
-        elif is_timedelta64_dtype(right) and not is_scalar(right):
-            # i.e. exclude np.timedelta64 object
+        elif is_timedelta64_dtype(right):
+            # We should only get here with non-scalar or timedelta64('NaT')
+            #  values for right
             # Note: we cannot use dispatch_to_index_op because
-            # that may incorrectly raise TypeError when we
-            # should get NullFrequencyError
+            #  that may incorrectly raise TypeError when we
+            #  should get NullFrequencyError
             result = op(pd.Index(left), right)
             return construct_result(left, result,
                                     index=left.index, name=res_name,
@@ -1941,8 +1949,7 @@ def f(self, other):
 
             # straight boolean comparisons we want to allow all columns
             # (regardless of dtype to pass thru) See #4537 for discussion.
-            res = self._combine_const(other, func,
-                                      errors='ignore')
+            res = self._combine_const(other, func)
             return res.fillna(True).astype(bool)
 
     f.__name__ = op_name

pandas/core/sparse/frame.py

@@ -620,7 +620,7 @@ def _combine_match_columns(self, other, func, level=None):
             new_data, index=left.index, columns=left.columns,
             default_fill_value=self.default_fill_value).__finalize__(self)
 
-    def _combine_const(self, other, func, errors='raise'):
+    def _combine_const(self, other, func):
         return self._apply_columns(lambda x: func(x, other))
 
     def _get_op_result_fill_value(self, other, func):

pandas/tests/arithmetic/conftest.py

@@ -70,7 +70,8 @@ def scalar_td(request):
                         pd.Timedelta(days=3).to_pytimedelta(),
                         pd.Timedelta('72:00:00'),
                         np.timedelta64(3, 'D'),
-                        np.timedelta64(72, 'h')])
+                        np.timedelta64(72, 'h')],
+                ids=lambda x: type(x).__name__)
 def three_days(request):
     """
     Several timedelta-like and DateOffset objects that each represent
@@ -84,7 +85,8 @@ def three_days(request):
                         pd.Timedelta(hours=2).to_pytimedelta(),
                         pd.Timedelta(seconds=2 * 3600),
                         np.timedelta64(2, 'h'),
-                        np.timedelta64(120, 'm')])
+                        np.timedelta64(120, 'm')],
+                ids=lambda x: type(x).__name__)
 def two_hours(request):
     """
     Several timedelta-like and DateOffset objects that each represent

pandas/tests/arithmetic/test_numeric.py

@@ -148,15 +148,11 @@ def test_numeric_arr_mul_tdscalar(self, scalar_td, numeric_idx, box):
         tm.assert_equal(commute, expected)
 
     def test_numeric_arr_rdiv_tdscalar(self, three_days, numeric_idx, box):
-        index = numeric_idx[1:3]
 
-        broken = (isinstance(three_days, np.timedelta64) and
-                  three_days.dtype != 'm8[ns]')
-        broken = broken or isinstance(three_days, pd.offsets.Tick)
-        if box is not pd.Index and broken:
-            # np.timedelta64(3, 'D') / 2 == np.timedelta64(1, 'D')
-            raise pytest.xfail("timedelta64 not converted to nanos; "
-                               "Tick division not implemented")
+        if box is not pd.Index and isinstance(three_days, pd.offsets.Tick):
+            raise pytest.xfail("Tick division not implemented")
+
+        index = numeric_idx[1:3]
 
         expected = TimedeltaIndex(['3 Days', '36 Hours'])
 
@@ -169,6 +165,26 @@ def test_numeric_arr_rdiv_tdscalar(self, three_days, numeric_idx, box):
         with pytest.raises(TypeError):
             index / three_days
 
+    @pytest.mark.parametrize('other', [
+        pd.Timedelta(hours=31),
+        pd.Timedelta(hours=31).to_pytimedelta(),
+        pd.Timedelta(hours=31).to_timedelta64(),
+        pd.Timedelta(hours=31).to_timedelta64().astype('m8[h]'),
+        np.timedelta64('NaT'),
+        np.timedelta64('NaT', 'D'),
+        pd.offsets.Minute(3),
+        pd.offsets.Second(0)])
+    def test_add_sub_timedeltalike_invalid(self, numeric_idx, other, box):
+        left = tm.box_expected(numeric_idx, box)
+        with pytest.raises(TypeError):
+            left + other
+        with pytest.raises(TypeError):
+            other + left
+        with pytest.raises(TypeError):
+            left - other
+        with pytest.raises(TypeError):
+            other - left
+
 
 # ------------------------------------------------------------------
 # Arithmetic

pandas/tests/arithmetic/test_timedelta64.py

@@ -1051,10 +1051,8 @@ def test_tdi_mul_float_series(self, box_df_fail):
         pd.Float64Index(range(1, 11)),
         pd.RangeIndex(1, 11)
     ], ids=lambda x: type(x).__name__)
-    def test_tdi_rmul_arraylike(self, other, box_df_fail):
-        # RangeIndex fails to return NotImplemented, for others
-        # DataFrame tries to broadcast incorrectly
-        box = box_df_fail
+    def test_tdi_rmul_arraylike(self, other, box_df_broadcast_failure):
+        box = box_df_broadcast_failure
 
         tdi = TimedeltaIndex(['1 Day'] * 10)
         expected = timedelta_range('1 days', '10 days')

pandas/tests/indexes/interval/test_interval_tree.py

@@ -14,10 +14,22 @@ def dtype(request):
     return request.param
 
 
-@pytest.fixture(scope='class')
-def tree(dtype):
-    left = np.arange(5, dtype=dtype)
-    return IntervalTree(left, left + 2)
+@pytest.fixture(params=[1, 2, 10])
+def leaf_size(request):
+    return request.param
+
+
+@pytest.fixture(params=[
+    np.arange(5, dtype='int64'),
+    np.arange(5, dtype='int32'),
+    np.arange(5, dtype='uint64'),
+    np.arange(5, dtype='float64'),
+    np.arange(5, dtype='float32'),
+    np.array([0, 1, 2, 3, 4, np.nan], dtype='float64'),
+    np.array([0, 1, 2, 3, 4, np.nan], dtype='float32')])
+def tree(request, leaf_size):
+    left = request.param
+    return IntervalTree(left, left + 2, leaf_size=leaf_size)
 
 
 class TestIntervalTree(object):

pandas/tests/indexes/test_range.py

@@ -185,6 +185,25 @@ def test_constructor_name(self):
         assert copy.name == 'copy'
         assert new.name == 'new'
 
+    # TODO: mod, divmod?
+    @pytest.mark.parametrize('op', [operator.add, operator.sub,
+                                    operator.mul, operator.floordiv,
+                                    operator.truediv, operator.pow])
+    def test_arithmetic_with_frame_or_series(self, op):
+        # check that we return NotImplemented when operating with Series
+        # or DataFrame
+        index = pd.RangeIndex(5)
+        other = pd.Series(np.random.randn(5))
+
+        expected = op(pd.Series(index), other)
+        result = op(index, other)
+        tm.assert_series_equal(result, expected)
+
+        other = pd.DataFrame(np.random.randn(2, 5))
+        expected = op(pd.DataFrame([index, index]), other)
+        result = op(index, other)
+        tm.assert_frame_equal(result, expected)
+
     def test_numeric_compat2(self):
         # validate that we are handling the RangeIndex overrides to numeric ops
         # and returning RangeIndex where possible

pandas/tests/internals/test_internals.py

@@ -1243,7 +1243,6 @@ def test_binop_other(self, op, value, dtype):
                    (operator.mul, '<M8[ns]'),
                    (operator.add, '<M8[ns]'),
                    (operator.pow, '<m8[ns]'),
-                   (operator.mod, '<m8[ns]'),
                    (operator.mul, '<m8[ns]')}
 
         if (op, dtype) in invalid: