From a910a3310d1fbc11e6f5d0bf926df8d8782c77c4 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Mon, 7 Jan 2019 10:45:59 -0800 Subject: [PATCH 1/2] fix to_datetime failing to raise on mixed tznaive/tzaware datetimes --- doc/source/whatsnew/v0.24.0.rst | 1 + pandas/_libs/tslibs/conversion.pyx | 5 +++++ pandas/tests/arrays/test_array.py | 6 +----- pandas/tests/arrays/test_datetimes.py | 19 +++++++++++++++++++ .../indexes/datetimes/test_construction.py | 15 +++------------ 5 files changed, 29 insertions(+), 17 deletions(-) diff --git a/doc/source/whatsnew/v0.24.0.rst b/doc/source/whatsnew/v0.24.0.rst index 4bc50695e1ecd..797c3faeb1fd0 100644 --- a/doc/source/whatsnew/v0.24.0.rst +++ b/doc/source/whatsnew/v0.24.0.rst @@ -1549,6 +1549,7 @@ Datetimelike - Bug in :class:`PeriodIndex` where comparisons against an array-like object with length 1 failed to raise ``ValueError`` (:issue:`23078`) - Bug in :meth:`DatetimeIndex.astype`, :meth:`PeriodIndex.astype` and :meth:`TimedeltaIndex.astype` ignoring the sign of the ``dtype`` for unsigned integer dtypes (:issue:`24405`). - Fixed bug in :meth:`Series.max` with ``datetime64[ns]``-dtype failing to return ``NaT`` when nulls are present and ``skipna=False`` is passed (:issue:`24265`) +- Bug in :func:`to_datetime` where arrays of ``datetime`` objects containing both timezone-aware and timezone-naive ``datetimes`` would fail to raise ``ValueError`` (:issue:`24569`) Timedelta ^^^^^^^^^ diff --git a/pandas/_libs/tslibs/conversion.pyx b/pandas/_libs/tslibs/conversion.pyx index 6aa02ca1e5421..6c8b732928bc3 100644 --- a/pandas/_libs/tslibs/conversion.pyx +++ b/pandas/_libs/tslibs/conversion.pyx @@ -167,6 +167,7 @@ def datetime_to_datetime64(values: object[:]): int64_t[:] iresult npy_datetimestruct dts _TSObject _ts + bint found_naive = False result = np.empty(n, dtype='M8[ns]') iresult = result.view('i8') @@ -176,6 +177,9 @@ def datetime_to_datetime64(values: object[:]): iresult[i] = NPY_NAT elif PyDateTime_Check(val): if val.tzinfo is not None: + if found_naive: + raise ValueError('Cannot mix tz-aware with ' + 'tz-naive values') if inferred_tz is not None: if not tz_compare(val.tzinfo, inferred_tz): raise ValueError('Array must be all same time zone') @@ -186,6 +190,7 @@ def datetime_to_datetime64(values: object[:]): iresult[i] = _ts.value check_dts_bounds(&_ts.dts) else: + found_naive = True if inferred_tz is not None: raise ValueError('Cannot mix tz-aware with ' 'tz-naive values') diff --git a/pandas/tests/arrays/test_array.py b/pandas/tests/arrays/test_array.py index 1d09a1f65e43f..0698b43b16c05 100644 --- a/pandas/tests/arrays/test_array.py +++ b/pandas/tests/arrays/test_array.py @@ -151,11 +151,7 @@ def test_array_inference(data, expected): [pd.Timestamp("2000", tz="CET"), pd.Timestamp("2000", tz="UTC")], # Mix of tz-aware and tz-naive [pd.Timestamp("2000", tz="CET"), pd.Timestamp("2000")], - # GH-24569 - pytest.param( - np.array([pd.Timestamp('2000'), pd.Timestamp('2000', tz='CET')]), - marks=pytest.mark.xfail(reason="bug in DTA._from_sequence") - ), + np.array([pd.Timestamp('2000'), pd.Timestamp('2000', tz='CET')]), ]) def test_array_inference_fails(data): result = pd.array(data) diff --git a/pandas/tests/arrays/test_datetimes.py b/pandas/tests/arrays/test_datetimes.py index 8d0e4f5a90557..ae267ad8f9ea4 100644 --- a/pandas/tests/arrays/test_datetimes.py +++ b/pandas/tests/arrays/test_datetimes.py @@ -16,6 +16,25 @@ class TestDatetimeArrayConstructor(object): + def test_mixing_naive_tzaware_raises(self): + # GH#24569 + arr = np.array([pd.Timestamp('2000'), pd.Timestamp('2000', tz='CET')]) + + msg = ('Cannot mix tz-aware with tz-naive values|' + 'Tz-aware datetime.datetime cannot be converted ' + 'to datetime64 unless utc=True') + + for obj in [arr, arr[::-1]]: + # check that we raise regardless of whether naive is found + # before aware or vice-versa + for meth in [DatetimeArray._from_sequence, + sequence_to_dt64ns, + pd.to_datetime, + pd.DatetimeIndex]: + + with pytest.raises(ValueError, match=msg): + meth(obj) + def test_from_pandas_array(self): arr = pd.array(np.arange(5, dtype=np.int64)) * 3600 * 10**9 diff --git a/pandas/tests/indexes/datetimes/test_construction.py b/pandas/tests/indexes/datetimes/test_construction.py index 97de4cd98dedf..07c42afe44b33 100644 --- a/pandas/tests/indexes/datetimes/test_construction.py +++ b/pandas/tests/indexes/datetimes/test_construction.py @@ -306,16 +306,6 @@ def test_construction_dti_with_mixed_timezones(self): tm.assert_index_equal(result, exp, exact=True) assert isinstance(result, DatetimeIndex) - # different tz coerces tz-naive to tz-awareIndex(dtype=object) - result = DatetimeIndex([Timestamp('2011-01-01 10:00'), - Timestamp('2011-01-02 10:00', - tz='US/Eastern')], name='idx') - exp = DatetimeIndex([Timestamp('2011-01-01 05:00'), - Timestamp('2011-01-02 10:00')], - tz='US/Eastern', name='idx') - tm.assert_index_equal(result, exp, exact=True) - assert isinstance(result, DatetimeIndex) - # tz mismatch affecting to tz-aware raises TypeError/ValueError with pytest.raises(ValueError): @@ -323,7 +313,8 @@ def test_construction_dti_with_mixed_timezones(self): Timestamp('2011-01-02 10:00', tz='US/Eastern')], name='idx') - with pytest.raises(TypeError, match='data is already tz-aware'): + msg = 'cannot be converted to datetime64' + with pytest.raises(ValueError, match=msg): DatetimeIndex([Timestamp('2011-01-01 10:00'), Timestamp('2011-01-02 10:00', tz='US/Eastern')], tz='Asia/Tokyo', name='idx') @@ -333,7 +324,7 @@ def test_construction_dti_with_mixed_timezones(self): Timestamp('2011-01-02 10:00', tz='US/Eastern')], tz='US/Eastern', name='idx') - with pytest.raises(TypeError, match='data is already tz-aware'): + with pytest.raises(ValueError, match=msg): # passing tz should results in DatetimeIndex, then mismatch raises # TypeError Index([pd.NaT, Timestamp('2011-01-01 10:00'), From 72e46b2fd901110326d494036d0299768f27ba68 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Mon, 7 Jan 2019 13:40:10 -0800 Subject: [PATCH 2/2] parametrize --- pandas/tests/arrays/test_datetimes.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/pandas/tests/arrays/test_datetimes.py b/pandas/tests/arrays/test_datetimes.py index ae267ad8f9ea4..8228ed7652fea 100644 --- a/pandas/tests/arrays/test_datetimes.py +++ b/pandas/tests/arrays/test_datetimes.py @@ -16,7 +16,11 @@ class TestDatetimeArrayConstructor(object): - def test_mixing_naive_tzaware_raises(self): + @pytest.mark.parametrize('meth', [DatetimeArray._from_sequence, + sequence_to_dt64ns, + pd.to_datetime, + pd.DatetimeIndex]) + def test_mixing_naive_tzaware_raises(self, meth): # GH#24569 arr = np.array([pd.Timestamp('2000'), pd.Timestamp('2000', tz='CET')]) @@ -27,13 +31,8 @@ def test_mixing_naive_tzaware_raises(self): for obj in [arr, arr[::-1]]: # check that we raise regardless of whether naive is found # before aware or vice-versa - for meth in [DatetimeArray._from_sequence, - sequence_to_dt64ns, - pd.to_datetime, - pd.DatetimeIndex]: - - with pytest.raises(ValueError, match=msg): - meth(obj) + with pytest.raises(ValueError, match=msg): + meth(obj) def test_from_pandas_array(self): arr = pd.array(np.arange(5, dtype=np.int64)) * 3600 * 10**9