From e7130918361330c8397d23a2282efc8dbdd91ed0 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Wed, 11 Jul 2018 08:15:02 -0700 Subject: [PATCH 1/5] fix Series.hasnans to not cache #19700 --- doc/source/whatsnew/v0.24.0.txt | 1 + pandas/core/series.py | 4 ++++ pandas/tests/series/test_internals.py | 14 ++++++++++++++ 3 files changed, 19 insertions(+) diff --git a/doc/source/whatsnew/v0.24.0.txt b/doc/source/whatsnew/v0.24.0.txt index 034a56b2ac0cb..2d39f76f9acc0 100644 --- a/doc/source/whatsnew/v0.24.0.txt +++ b/doc/source/whatsnew/v0.24.0.txt @@ -369,6 +369,7 @@ Missing ^^^^^^^ - Bug in :func:`DataFrame.fillna` where a ``ValueError`` would raise when one column contained a ``datetime64[ns, tz]`` dtype (:issue:`15522`) +- Bug in :func:`Series.hasnans` that could be incorrectly cached and return incorrect answers if null elements are introduced after an initial call (:issue:`19700`) MultiIndex ^^^^^^^^^^ diff --git a/pandas/core/series.py b/pandas/core/series.py index a63c4be98f738..0bdb9d9cc23a6 100644 --- a/pandas/core/series.py +++ b/pandas/core/series.py @@ -166,6 +166,10 @@ class Series(base.IndexOpsMixin, generic.NDFrame): ['asobject', 'sortlevel', 'reshape', 'get_value', 'set_value', 'from_csv', 'valid']) + # Override cache_readonly bc Series is mutable + hasnans = property(base.IndexOpsMixin.hasnans.func, + doc=base.IndexOpsMixin.hasnans.__doc__) + def __init__(self, data=None, index=None, dtype=None, name=None, copy=False, fastpath=False): diff --git a/pandas/tests/series/test_internals.py b/pandas/tests/series/test_internals.py index 79e23459ac992..506e7e14ffc4f 100644 --- a/pandas/tests/series/test_internals.py +++ b/pandas/tests/series/test_internals.py @@ -11,6 +11,7 @@ from pandas import Series from pandas.core.indexes.datetimes import Timestamp import pandas._libs.lib as lib +import pandas as pd from pandas.util.testing import assert_series_equal import pandas.util.testing as tm @@ -309,3 +310,16 @@ def test_convert_preserve_all_bool(self): r = s._convert(datetime=True, numeric=True) e = Series([False, True, False, False], dtype=bool) tm.assert_series_equal(r, e) + + +def test_hasnans_unchached_for_series(): + # GH#19700 + idx = pd.Index([0, 1]) + assert not idx.hasnans + assert 'hasnans' in idx._cache + ser = idx.to_series() + assert not ser.hasnans + assert not hasattr(ser, '_cache') + ser.iloc[-1] = np.nan + assert ser.hasnans + assert pd.Series.hasnans.__doc__ == pd.Index.hasnans.__doc__ From 4cc77f7dc10657746b08918d53d6581ab307b625 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Wed, 11 Jul 2018 08:16:48 -0700 Subject: [PATCH 2/5] Fix is_list_like on zero-dim arrays #19011 --- pandas/core/dtypes/inference.py | 7 ++++++- pandas/tests/dtypes/test_inference.py | 5 +++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/pandas/core/dtypes/inference.py b/pandas/core/dtypes/inference.py index d747e69d1ff39..a0456630c9a0f 100644 --- a/pandas/core/dtypes/inference.py +++ b/pandas/core/dtypes/inference.py @@ -278,10 +278,15 @@ def is_list_like(obj): False >>> is_list_like(1) False + >>> is_list_like(np.array([2])) + True + >>> is_list_like(np.array(2))) + False """ return (isinstance(obj, Iterable) and - not isinstance(obj, string_and_binary_types)) + not isinstance(obj, string_and_binary_types) and + not (isinstance(obj, np.ndarray) and obj.ndim == 0)) def is_array_like(obj): diff --git a/pandas/tests/dtypes/test_inference.py b/pandas/tests/dtypes/test_inference.py index 65527ac1b278f..f81767156b255 100644 --- a/pandas/tests/dtypes/test_inference.py +++ b/pandas/tests/dtypes/test_inference.py @@ -67,13 +67,14 @@ def __getitem__(self): [ [], [1], (1, ), (1, 2), {'a': 1}, set([1, 'a']), Series([1]), - Series([]), Series(['a']).str]) + Series([]), Series(['a']).str, + np.array([2])]) def test_is_list_like_passes(ll): assert inference.is_list_like(ll) @pytest.mark.parametrize( - "ll", [1, '2', object(), str]) + "ll", [1, '2', object(), str, np.array(2)]) def test_is_list_like_fails(ll): assert not inference.is_list_like(ll) From cee2fc175ab54b40e781cc0e53639b909ce164c5 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Wed, 11 Jul 2018 08:43:03 -0700 Subject: [PATCH 3/5] fix Timestamp.resolution #21336 --- doc/source/whatsnew/v0.24.0.txt | 1 + pandas/_libs/tslibs/timestamps.pyx | 9 +++++++++ pandas/tests/scalar/timestamp/test_timestamp.py | 5 +++++ 3 files changed, 15 insertions(+) diff --git a/doc/source/whatsnew/v0.24.0.txt b/doc/source/whatsnew/v0.24.0.txt index 2d39f76f9acc0..00379c7e9d511 100644 --- a/doc/source/whatsnew/v0.24.0.txt +++ b/doc/source/whatsnew/v0.24.0.txt @@ -301,6 +301,7 @@ Datetimelike ^^^^^^^^^^^^ - Fixed bug where two :class:`DateOffset` objects with different ``normalize`` attributes could evaluate as equal (:issue:`21404`) +- Fixed bug where :meth:`Timestamp.resolution` incorrectly returned 1-microsecond ``timedelta`` instead of 1-nanosecond :class:`Timedelta` (:issue:`21336`,:issue:`21365`) Timedelta ^^^^^^^^^ diff --git a/pandas/_libs/tslibs/timestamps.pyx b/pandas/_libs/tslibs/timestamps.pyx index 711db7cc8fbe2..864950ff03eae 100644 --- a/pandas/_libs/tslibs/timestamps.pyx +++ b/pandas/_libs/tslibs/timestamps.pyx @@ -407,6 +407,15 @@ cdef class _Timestamp(datetime): def asm8(self): return np.datetime64(self.value, 'ns') + @property + def resolution(self): + """ + Return resolution describing the smallest difference between two + times that can be represented by Timestamp object_state + """ + # GH#21336, GH#21365 + return Timedelta(nanoseconds=1) + def timestamp(self): """Return POSIX timestamp as float.""" # py27 compat, see GH#17329 diff --git a/pandas/tests/scalar/timestamp/test_timestamp.py b/pandas/tests/scalar/timestamp/test_timestamp.py index 5272059163a07..4172bfd41b9db 100644 --- a/pandas/tests/scalar/timestamp/test_timestamp.py +++ b/pandas/tests/scalar/timestamp/test_timestamp.py @@ -172,6 +172,11 @@ def test_woy_boundary(self): 2005, 1, 1), (2005, 1, 2)]]) assert (result == [52, 52, 53, 53]).all() + def test_resolution(self): + # GH#21336, GH#21365 + dt = Timestamp('2100-01-01 00:00:00') + assert dt.resolution == Timedelta(nanoseconds=1) + class TestTimestampConstructors(object): From 6e397110d1e6c691df41e903cc87f3ed7dc7c015 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Wed, 11 Jul 2018 09:25:43 -0700 Subject: [PATCH 4/5] fix problems caused by incorrect uses of is_list_like --- pandas/core/indexes/datetimes.py | 5 ++--- pandas/core/tools/timedeltas.py | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py index b8a89ac26c9d9..217bb3e7d1734 100644 --- a/pandas/core/indexes/datetimes.py +++ b/pandas/core/indexes/datetimes.py @@ -93,7 +93,7 @@ def _dt_index_cmp(opname, cls): def wrapper(self, other): func = getattr(super(DatetimeIndex, self), opname) - if isinstance(other, (datetime, compat.string_types)): + if isinstance(other, (datetime, np.datetime64, compat.string_types)): if isinstance(other, datetime): # GH#18435 strings get a pass from tzawareness compat self._assert_tzawareness_compat(other) @@ -105,8 +105,7 @@ def wrapper(self, other): else: if isinstance(other, list): other = DatetimeIndex(other) - elif not isinstance(other, (np.datetime64, np.ndarray, - Index, ABCSeries)): + elif not isinstance(other, (np.ndarray, Index, ABCSeries)): # Following Timestamp convention, __eq__ is all-False # and __ne__ is all True, others raise TypeError. if opname == '__eq__': diff --git a/pandas/core/tools/timedeltas.py b/pandas/core/tools/timedeltas.py index 29618fb4dec52..ed2659973cc6a 100644 --- a/pandas/core/tools/timedeltas.py +++ b/pandas/core/tools/timedeltas.py @@ -85,7 +85,7 @@ def to_timedelta(arg, unit='ns', box=True, errors='raise'): elif isinstance(arg, ABCIndexClass): return _convert_listlike(arg, unit=unit, box=box, errors=errors, name=arg.name) - elif is_list_like(arg) and getattr(arg, 'ndim', 1) == 0: + elif isinstance(arg, np.ndarray) and arg.ndim == 0: # extract array scalar and process below arg = arg.item() elif is_list_like(arg) and getattr(arg, 'ndim', 1) == 1: From 84e96632473142a380ec1f5e0cd999fcc6181b14 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Wed, 11 Jul 2018 10:47:21 -0700 Subject: [PATCH 5/5] Fix prep_ndarray is_list_like usage --- pandas/core/frame.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pandas/core/frame.py b/pandas/core/frame.py index 4e6ddf64145a8..6380944338010 100644 --- a/pandas/core/frame.py +++ b/pandas/core/frame.py @@ -7679,6 +7679,9 @@ def convert(v): try: if is_list_like(values[0]) or hasattr(values[0], 'len'): values = np.array([convert(v) for v in values]) + elif isinstance(values[0], np.ndarray) and values[0].ndim == 0: + # GH#21861 + values = np.array([convert(v) for v in values]) else: values = convert(values) except: