Skip to content

COMPAT: remove NaT comparison warnings with numpy >= 1.11 #12969

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

COMPAT: remove NaT comparison warnings with numpy >= 1.11 #12969

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions doc/source/whatsnew/v0.18.1.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -281,6 +281,7 @@ Bug Fixes
- Bug in ``.astype()`` of a ``Float64Inde/Int64Index`` to an ``Int64Index`` (:issue:`12881`)
- Bug in roundtripping an integer based index in ``.to_json()/.read_json()`` when ``orient='index'`` (the default) (:issue:`12866`)

- Compat with >= numpy 1.11 for NaT comparions (:issue:`12969`)
- Bug in ``.drop()`` with a non-unique ``MultiIndex``. (:issue:`12701`)
- Bug in ``.concat`` of datetime tz-aware and naive DataFrames (:issue:`12467`)
- Bug in correctly raising a ``ValueError`` in ``.resample(..).fillna(..)`` when passing a non-string (:issue:`12952`)
Expand Down
12 changes: 11 additions & 1 deletion pandas/indexes/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,8 @@
is_iterator, is_categorical_dtype,
_ensure_object, _ensure_int64, is_bool_indexer,
is_list_like, is_bool_dtype,
is_integer_dtype, is_float_dtype)
is_integer_dtype, is_float_dtype,
needs_i8_conversion)
from pandas.core.strings import StringAccessorMixin

from pandas.core.config import get_option
Expand Down Expand Up @@ -3068,6 +3069,9 @@ def _evaluate_with_timedelta_like(self, other, op, opstr):
def _evaluate_with_datetime_like(self, other, op, opstr):
raise TypeError("can only perform ops with datetime like values")

def _evalute_compare(self, op):
raise base.AbstractMethodError(self)

@classmethod
def _add_comparison_methods(cls):
""" add in comparison methods """
Expand All @@ -3077,6 +3081,12 @@ def _evaluate_compare(self, other):
if isinstance(other, (np.ndarray, Index, ABCSeries)):
if other.ndim > 0 and len(self) != len(other):
raise ValueError('Lengths must match to compare')

# we may need to directly compare underlying
# representations
if needs_i8_conversion(self) and needs_i8_conversion(other):
return self._evaluate_compare(other, op)

func = getattr(self.values, op)
result = func(np.asarray(other))

Expand Down
3 changes: 2 additions & 1 deletion pandas/tests/test_lib.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -223,7 +223,8 @@ def test_isscalar_numpy_array_scalars(self):
def test_isscalar_numpy_zerodim_arrays(self):
for zerodim in [np.array(1), np.array('foobar'),
np.array(np.datetime64('2014-01-01')),
np.array(np.timedelta64(1, 'h'))]:
np.array(np.timedelta64(1, 'h')),
np.array(np.datetime64('NaT'))]:
self.assertFalse(lib.isscalar(zerodim))
self.assertTrue(lib.isscalar(lib.item_from_zerodim(zerodim)))

Expand Down
35 changes: 34 additions & 1 deletion pandas/tseries/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@
from pandas import compat
import numpy as np
from pandas.core import common as com, algorithms
from pandas.core.common import is_integer, is_float, AbstractMethodError
from pandas.core.common import (is_integer, is_float, is_bool_dtype,
AbstractMethodError)
import pandas.formats.printing as printing
import pandas.tslib as tslib
import pandas.lib as lib
Expand Down Expand Up @@ -124,6 +125,38 @@ def wrapper(left, right):

return wrapper

def _evaluate_compare(self, other, op):
"""
We have been called because a comparison between
8 aware arrays. numpy >= 1.11 will
now warn about NaT comparisons
"""

# coerce to a similar object
if not isinstance(other, type(self)):
if not com.is_list_like(other):
# scalar
other = [other]
elif lib.isscalar(lib.item_from_zerodim(other)):
# ndarray scalar
other = [other.item()]
other = type(self)(other)

# compare
result = getattr(self.asi8, op)(other.asi8)

# technically we could support bool dtyped Index
# for now just return the indexing array directly
mask = (self._isnan) | (other._isnan)
if is_bool_dtype(result):
result[mask] = False
return result
try:
result[mask] = tslib.iNaT
return Index(result)
except TypeError:
return result

@property
def _box_func(self):
"""
Expand Down
115 changes: 59 additions & 56 deletions pandas/tseries/tests/test_timeseries.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2532,74 +2532,77 @@ def test_comparisons_nat(self):
cases = [(fidx1, fidx2), (didx1, didx2), (didx1, darr)]

# Check pd.NaT is handles as the same as np.nan
for idx1, idx2 in cases:
with tm.assert_produces_warning(None):
for idx1, idx2 in cases:

result = idx1 < idx2
expected = np.array([True, False, False, False, True, False])
self.assert_numpy_array_equal(result, expected)
result = idx1 < idx2
expected = np.array([True, False, False, False, True, False])
self.assert_numpy_array_equal(result, expected)

result = idx2 > idx1
expected = np.array([True, False, False, False, True, False])
self.assert_numpy_array_equal(result, expected)
result = idx2 > idx1
expected = np.array([True, False, False, False, True, False])
self.assert_numpy_array_equal(result, expected)

result = idx1 <= idx2
expected = np.array([True, False, False, False, True, True])
self.assert_numpy_array_equal(result, expected)
result = idx1 <= idx2
expected = np.array([True, False, False, False, True, True])
self.assert_numpy_array_equal(result, expected)

result = idx2 >= idx1
expected = np.array([True, False, False, False, True, True])
self.assert_numpy_array_equal(result, expected)
result = idx2 >= idx1
expected = np.array([True, False, False, False, True, True])
self.assert_numpy_array_equal(result, expected)

result = idx1 == idx2
expected = np.array([False, False, False, False, False, True])
self.assert_numpy_array_equal(result, expected)
result = idx1 == idx2
expected = np.array([False, False, False, False, False, True])
self.assert_numpy_array_equal(result, expected)

result = idx1 != idx2
expected = np.array([True, True, True, True, True, False])
self.assert_numpy_array_equal(result, expected)
result = idx1 != idx2
expected = np.array([True, True, True, True, True, False])
self.assert_numpy_array_equal(result, expected)

for idx1, val in [(fidx1, np.nan), (didx1, pd.NaT)]:
result = idx1 < val
expected = np.array([False, False, False, False, False, False])
self.assert_numpy_array_equal(result, expected)
result = idx1 > val
self.assert_numpy_array_equal(result, expected)
with tm.assert_produces_warning(None):
for idx1, val in [(fidx1, np.nan), (didx1, pd.NaT)]:
result = idx1 < val
expected = np.array([False, False, False, False, False, False])
self.assert_numpy_array_equal(result, expected)
result = idx1 > val
self.assert_numpy_array_equal(result, expected)

result = idx1 <= val
self.assert_numpy_array_equal(result, expected)
result = idx1 >= val
self.assert_numpy_array_equal(result, expected)
result = idx1 <= val
self.assert_numpy_array_equal(result, expected)
result = idx1 >= val
self.assert_numpy_array_equal(result, expected)

result = idx1 == val
self.assert_numpy_array_equal(result, expected)
result = idx1 == val
self.assert_numpy_array_equal(result, expected)

result = idx1 != val
expected = np.array([True, True, True, True, True, True])
self.assert_numpy_array_equal(result, expected)
result = idx1 != val
expected = np.array([True, True, True, True, True, True])
self.assert_numpy_array_equal(result, expected)

# Check pd.NaT is handles as the same as np.nan
for idx1, val in [(fidx1, 3), (didx1, datetime(2014, 3, 1))]:
result = idx1 < val
expected = np.array([True, False, False, False, False, False])
self.assert_numpy_array_equal(result, expected)
result = idx1 > val
expected = np.array([False, False, False, False, True, True])
self.assert_numpy_array_equal(result, expected)

result = idx1 <= val
expected = np.array([True, False, True, False, False, False])
self.assert_numpy_array_equal(result, expected)
result = idx1 >= val
expected = np.array([False, False, True, False, True, True])
self.assert_numpy_array_equal(result, expected)

result = idx1 == val
expected = np.array([False, False, True, False, False, False])
self.assert_numpy_array_equal(result, expected)

result = idx1 != val
expected = np.array([True, True, False, True, True, True])
self.assert_numpy_array_equal(result, expected)
with tm.assert_produces_warning(None):
for idx1, val in [(fidx1, 3), (didx1, datetime(2014, 3, 1))]:
result = idx1 < val
expected = np.array([True, False, False, False, False, False])
self.assert_numpy_array_equal(result, expected)
result = idx1 > val
expected = np.array([False, False, False, False, True, True])
self.assert_numpy_array_equal(result, expected)

result = idx1 <= val
expected = np.array([True, False, True, False, False, False])
self.assert_numpy_array_equal(result, expected)
result = idx1 >= val
expected = np.array([False, False, True, False, True, True])
self.assert_numpy_array_equal(result, expected)

result = idx1 == val
expected = np.array([False, False, True, False, False, False])
self.assert_numpy_array_equal(result, expected)

result = idx1 != val
expected = np.array([True, True, False, True, True, True])
self.assert_numpy_array_equal(result, expected)

def test_map(self):
rng = date_range('1/1/2000', periods=10)
Expand Down