diff --git a/doc/source/whatsnew/v0.18.1.txt b/doc/source/whatsnew/v0.18.1.txt index 60103024909a0..ea6315924010c 100644 --- a/doc/source/whatsnew/v0.18.1.txt +++ b/doc/source/whatsnew/v0.18.1.txt @@ -281,6 +281,7 @@ Bug Fixes - Bug in ``.astype()`` of a ``Float64Inde/Int64Index`` to an ``Int64Index`` (:issue:`12881`) - Bug in roundtripping an integer based index in ``.to_json()/.read_json()`` when ``orient='index'`` (the default) (:issue:`12866`) +- Compat with >= numpy 1.11 for NaT comparions (:issue:`12969`) - Bug in ``.drop()`` with a non-unique ``MultiIndex``. (:issue:`12701`) - Bug in ``.concat`` of datetime tz-aware and naive DataFrames (:issue:`12467`) - Bug in correctly raising a ``ValueError`` in ``.resample(..).fillna(..)`` when passing a non-string (:issue:`12952`) diff --git a/pandas/indexes/base.py b/pandas/indexes/base.py index 77e53f839f4f4..75f13226f4f50 100644 --- a/pandas/indexes/base.py +++ b/pandas/indexes/base.py @@ -28,7 +28,8 @@ is_iterator, is_categorical_dtype, _ensure_object, _ensure_int64, is_bool_indexer, is_list_like, is_bool_dtype, - is_integer_dtype, is_float_dtype) + is_integer_dtype, is_float_dtype, + needs_i8_conversion) from pandas.core.strings import StringAccessorMixin from pandas.core.config import get_option @@ -3068,6 +3069,9 @@ def _evaluate_with_timedelta_like(self, other, op, opstr): def _evaluate_with_datetime_like(self, other, op, opstr): raise TypeError("can only perform ops with datetime like values") + def _evalute_compare(self, op): + raise base.AbstractMethodError(self) + @classmethod def _add_comparison_methods(cls): """ add in comparison methods """ @@ -3077,6 +3081,12 @@ def _evaluate_compare(self, other): if isinstance(other, (np.ndarray, Index, ABCSeries)): if other.ndim > 0 and len(self) != len(other): raise ValueError('Lengths must match to compare') + + # we may need to directly compare underlying + # representations + if needs_i8_conversion(self) and needs_i8_conversion(other): + return self._evaluate_compare(other, op) + func = getattr(self.values, op) result = func(np.asarray(other)) diff --git a/pandas/tests/test_lib.py b/pandas/tests/test_lib.py index fc0030718f2c9..5b1c82f8ff5e7 100644 --- a/pandas/tests/test_lib.py +++ b/pandas/tests/test_lib.py @@ -223,7 +223,8 @@ def test_isscalar_numpy_array_scalars(self): def test_isscalar_numpy_zerodim_arrays(self): for zerodim in [np.array(1), np.array('foobar'), np.array(np.datetime64('2014-01-01')), - np.array(np.timedelta64(1, 'h'))]: + np.array(np.timedelta64(1, 'h')), + np.array(np.datetime64('NaT'))]: self.assertFalse(lib.isscalar(zerodim)) self.assertTrue(lib.isscalar(lib.item_from_zerodim(zerodim))) diff --git a/pandas/tseries/base.py b/pandas/tseries/base.py index f59a970fd9853..059c77d21b4df 100644 --- a/pandas/tseries/base.py +++ b/pandas/tseries/base.py @@ -8,7 +8,8 @@ from pandas import compat import numpy as np from pandas.core import common as com, algorithms -from pandas.core.common import is_integer, is_float, AbstractMethodError +from pandas.core.common import (is_integer, is_float, is_bool_dtype, + AbstractMethodError) import pandas.formats.printing as printing import pandas.tslib as tslib import pandas.lib as lib @@ -124,6 +125,38 @@ def wrapper(left, right): return wrapper + def _evaluate_compare(self, other, op): + """ + We have been called because a comparison between + 8 aware arrays. numpy >= 1.11 will + now warn about NaT comparisons + """ + + # coerce to a similar object + if not isinstance(other, type(self)): + if not com.is_list_like(other): + # scalar + other = [other] + elif lib.isscalar(lib.item_from_zerodim(other)): + # ndarray scalar + other = [other.item()] + other = type(self)(other) + + # compare + result = getattr(self.asi8, op)(other.asi8) + + # technically we could support bool dtyped Index + # for now just return the indexing array directly + mask = (self._isnan) | (other._isnan) + if is_bool_dtype(result): + result[mask] = False + return result + try: + result[mask] = tslib.iNaT + return Index(result) + except TypeError: + return result + @property def _box_func(self): """ diff --git a/pandas/tseries/tests/test_timeseries.py b/pandas/tseries/tests/test_timeseries.py index abcf4244ba91f..eea8cf934ee7b 100644 --- a/pandas/tseries/tests/test_timeseries.py +++ b/pandas/tseries/tests/test_timeseries.py @@ -2532,74 +2532,77 @@ def test_comparisons_nat(self): cases = [(fidx1, fidx2), (didx1, didx2), (didx1, darr)] # Check pd.NaT is handles as the same as np.nan - for idx1, idx2 in cases: + with tm.assert_produces_warning(None): + for idx1, idx2 in cases: - result = idx1 < idx2 - expected = np.array([True, False, False, False, True, False]) - self.assert_numpy_array_equal(result, expected) + result = idx1 < idx2 + expected = np.array([True, False, False, False, True, False]) + self.assert_numpy_array_equal(result, expected) - result = idx2 > idx1 - expected = np.array([True, False, False, False, True, False]) - self.assert_numpy_array_equal(result, expected) + result = idx2 > idx1 + expected = np.array([True, False, False, False, True, False]) + self.assert_numpy_array_equal(result, expected) - result = idx1 <= idx2 - expected = np.array([True, False, False, False, True, True]) - self.assert_numpy_array_equal(result, expected) + result = idx1 <= idx2 + expected = np.array([True, False, False, False, True, True]) + self.assert_numpy_array_equal(result, expected) - result = idx2 >= idx1 - expected = np.array([True, False, False, False, True, True]) - self.assert_numpy_array_equal(result, expected) + result = idx2 >= idx1 + expected = np.array([True, False, False, False, True, True]) + self.assert_numpy_array_equal(result, expected) - result = idx1 == idx2 - expected = np.array([False, False, False, False, False, True]) - self.assert_numpy_array_equal(result, expected) + result = idx1 == idx2 + expected = np.array([False, False, False, False, False, True]) + self.assert_numpy_array_equal(result, expected) - result = idx1 != idx2 - expected = np.array([True, True, True, True, True, False]) - self.assert_numpy_array_equal(result, expected) + result = idx1 != idx2 + expected = np.array([True, True, True, True, True, False]) + self.assert_numpy_array_equal(result, expected) - for idx1, val in [(fidx1, np.nan), (didx1, pd.NaT)]: - result = idx1 < val - expected = np.array([False, False, False, False, False, False]) - self.assert_numpy_array_equal(result, expected) - result = idx1 > val - self.assert_numpy_array_equal(result, expected) + with tm.assert_produces_warning(None): + for idx1, val in [(fidx1, np.nan), (didx1, pd.NaT)]: + result = idx1 < val + expected = np.array([False, False, False, False, False, False]) + self.assert_numpy_array_equal(result, expected) + result = idx1 > val + self.assert_numpy_array_equal(result, expected) - result = idx1 <= val - self.assert_numpy_array_equal(result, expected) - result = idx1 >= val - self.assert_numpy_array_equal(result, expected) + result = idx1 <= val + self.assert_numpy_array_equal(result, expected) + result = idx1 >= val + self.assert_numpy_array_equal(result, expected) - result = idx1 == val - self.assert_numpy_array_equal(result, expected) + result = idx1 == val + self.assert_numpy_array_equal(result, expected) - result = idx1 != val - expected = np.array([True, True, True, True, True, True]) - self.assert_numpy_array_equal(result, expected) + result = idx1 != val + expected = np.array([True, True, True, True, True, True]) + self.assert_numpy_array_equal(result, expected) # Check pd.NaT is handles as the same as np.nan - for idx1, val in [(fidx1, 3), (didx1, datetime(2014, 3, 1))]: - result = idx1 < val - expected = np.array([True, False, False, False, False, False]) - self.assert_numpy_array_equal(result, expected) - result = idx1 > val - expected = np.array([False, False, False, False, True, True]) - self.assert_numpy_array_equal(result, expected) - - result = idx1 <= val - expected = np.array([True, False, True, False, False, False]) - self.assert_numpy_array_equal(result, expected) - result = idx1 >= val - expected = np.array([False, False, True, False, True, True]) - self.assert_numpy_array_equal(result, expected) - - result = idx1 == val - expected = np.array([False, False, True, False, False, False]) - self.assert_numpy_array_equal(result, expected) - - result = idx1 != val - expected = np.array([True, True, False, True, True, True]) - self.assert_numpy_array_equal(result, expected) + with tm.assert_produces_warning(None): + for idx1, val in [(fidx1, 3), (didx1, datetime(2014, 3, 1))]: + result = idx1 < val + expected = np.array([True, False, False, False, False, False]) + self.assert_numpy_array_equal(result, expected) + result = idx1 > val + expected = np.array([False, False, False, False, True, True]) + self.assert_numpy_array_equal(result, expected) + + result = idx1 <= val + expected = np.array([True, False, True, False, False, False]) + self.assert_numpy_array_equal(result, expected) + result = idx1 >= val + expected = np.array([False, False, True, False, True, True]) + self.assert_numpy_array_equal(result, expected) + + result = idx1 == val + expected = np.array([False, False, True, False, False, False]) + self.assert_numpy_array_equal(result, expected) + + result = idx1 != val + expected = np.array([True, True, False, True, True, True]) + self.assert_numpy_array_equal(result, expected) def test_map(self): rng = date_range('1/1/2000', periods=10)