From 1d52c6059ec85036c08f8f4feff3865b066d13e0 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Sun, 23 Dec 2018 23:42:31 -0800 Subject: [PATCH 1/3] BUG: to_datetime ignores argument combinations --- doc/source/whatsnew/v0.24.0.rst | 1 + pandas/core/tools/datetimes.py | 7 ++++++- pandas/tests/indexes/datetimes/test_tools.py | 7 +++++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v0.24.0.rst b/doc/source/whatsnew/v0.24.0.rst index a2abda019812a..1b038db524b81 100644 --- a/doc/source/whatsnew/v0.24.0.rst +++ b/doc/source/whatsnew/v0.24.0.rst @@ -1361,6 +1361,7 @@ Timezones - Bug in :class:`DatetimeIndex` constructor where ``NaT`` and ``dateutil.tz.tzlocal`` would raise an ``OutOfBoundsDatetime`` error (:issue:`23807`) - Bug in :meth:`DatetimeIndex.tz_localize` and :meth:`Timestamp.tz_localize` with ``dateutil.tz.tzlocal`` near a DST transition that would return an incorrectly localized datetime (:issue:`23807`) - Bug in :class:`Timestamp` constructor where a ``dateutil.tz.tzutc`` timezone passed with a ``datetime.datetime`` argument would be converted to a ``pytz.UTC`` timezone (:issue:`23807`) +- Bug in :func:`to_datetime` where ``utc=True`` was not respected when specifying a ``unit`` and ``errors=='ignore'`` (:issue:`23758`) Offsets ^^^^^^^ diff --git a/pandas/core/tools/datetimes.py b/pandas/core/tools/datetimes.py index 45d2615a3d055..ee295c8d722d0 100644 --- a/pandas/core/tools/datetimes.py +++ b/pandas/core/tools/datetimes.py @@ -204,7 +204,12 @@ def _convert_listlike_datetimes(arg, box, format, name=None, tz=None, if box: if errors == 'ignore': from pandas import Index - return Index(result, name=name) + result = Index(result, name=name) + # GH 23758: We may still need to localize the result with tz + try: + return result.tz_localize(tz) + except AttributeError: + return result return DatetimeIndex(result, tz=tz, name=name) return result diff --git a/pandas/tests/indexes/datetimes/test_tools.py b/pandas/tests/indexes/datetimes/test_tools.py index 13f9648d46216..39b8ef0c865a1 100644 --- a/pandas/tests/indexes/datetimes/test_tools.py +++ b/pandas/tests/indexes/datetimes/test_tools.py @@ -949,6 +949,13 @@ def test_dataframe_dtypes(self, cache): with pytest.raises(ValueError): to_datetime(df, cache=cache) + def test_to_datetime_errors_ignore_utc_true(self): + # GH 23758 + result = pd.to_datetime([1], unit='s', box=True, utc=True, + errors='ignore') + expected = DatetimeIndex(['1970-01-01 00:00:01'], tz='UTC') + tm.assert_index_equal(result, expected) + class TestToDatetimeMisc(object): def test_to_datetime_barely_out_of_bounds(self): From 8a543ada4b867f22f4feab58dbee6ba062b0017e Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Mon, 24 Dec 2018 00:02:51 -0800 Subject: [PATCH 2/3] don't ignore box or utc when passing unit mapping --- doc/source/whatsnew/v0.24.0.rst | 1 + pandas/core/tools/datetimes.py | 14 ++++++++++---- pandas/tests/indexes/datetimes/test_tools.py | 20 ++++++++++++++++++++ 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/doc/source/whatsnew/v0.24.0.rst b/doc/source/whatsnew/v0.24.0.rst index 1b038db524b81..39ed87134def1 100644 --- a/doc/source/whatsnew/v0.24.0.rst +++ b/doc/source/whatsnew/v0.24.0.rst @@ -1318,6 +1318,7 @@ Datetimelike - Bug in :class:`DatetimeIndex` where constructing a :class:`DatetimeIndex` from a :class:`Categorical` or :class:`CategoricalIndex` would incorrectly drop timezone information (:issue:`18664`) - Bug in :class:`DatetimeIndex` and :class:`TimedeltaIndex` where indexing with ``Ellipsis`` would incorrectly lose the index's ``freq`` attribute (:issue:`21282`) - Clarified error message produced when passing an incorrect ``freq`` argument to :class:`DatetimeIndex` with ``NaT`` as the first entry in the passed data (:issue:`11587`) +- Bug in :func:`to_datetime` where ``box`` and ``utc``arguments were ignored when passing a :class:`DataFrame` or ``dict`` of unit mappings (:issue:`23760`) Timedelta ^^^^^^^^^ diff --git a/pandas/core/tools/datetimes.py b/pandas/core/tools/datetimes.py index ee295c8d722d0..4df6f4c661d43 100644 --- a/pandas/core/tools/datetimes.py +++ b/pandas/core/tools/datetimes.py @@ -577,7 +577,7 @@ def to_datetime(arg, errors='raise', dayfirst=False, yearfirst=False, values = convert_listlike(arg._values, True, format) result = Series(values, index=arg.index, name=arg.name) elif isinstance(arg, (ABCDataFrame, compat.MutableMapping)): - result = _assemble_from_unit_mappings(arg, errors=errors) + result = _assemble_from_unit_mappings(arg, errors, box, tz) elif isinstance(arg, ABCIndexClass): cache_array = _maybe_cache(arg, format, cache, convert_listlike) if not cache_array.empty: @@ -623,7 +623,7 @@ def to_datetime(arg, errors='raise', dayfirst=False, yearfirst=False, } -def _assemble_from_unit_mappings(arg, errors): +def _assemble_from_unit_mappings(arg, errors, box, tz): """ assemble the unit specified fields from the arg (DataFrame) Return a Series for actual parsing @@ -636,6 +636,11 @@ def _assemble_from_unit_mappings(arg, errors): - If 'raise', then invalid parsing will raise an exception - If 'coerce', then invalid parsing will be set as NaT - If 'ignore', then invalid parsing will return the input + box : boolean + + - If True, return a DatetimeIndex + - If False, return an array + tz : None or 'utc' Returns ------- @@ -688,7 +693,7 @@ def coerce(values): coerce(arg[unit_rev['month']]) * 100 + coerce(arg[unit_rev['day']])) try: - values = to_datetime(values, format='%Y%m%d', errors=errors) + values = to_datetime(values, format='%Y%m%d', errors=errors, utc=tz) except (TypeError, ValueError) as e: raise ValueError("cannot assemble the " "datetimes: {error}".format(error=e)) @@ -703,7 +708,8 @@ def coerce(values): except (TypeError, ValueError) as e: raise ValueError("cannot assemble the datetimes [{value}]: " "{error}".format(value=value, error=e)) - + if not box: + return values.values return values diff --git a/pandas/tests/indexes/datetimes/test_tools.py b/pandas/tests/indexes/datetimes/test_tools.py index 39b8ef0c865a1..0abcf597f7212 100644 --- a/pandas/tests/indexes/datetimes/test_tools.py +++ b/pandas/tests/indexes/datetimes/test_tools.py @@ -949,6 +949,26 @@ def test_dataframe_dtypes(self, cache): with pytest.raises(ValueError): to_datetime(df, cache=cache) + def test_dataframe_box_false(self): + # GH 23760 + df = pd.DataFrame({'year': [2015, 2016], + 'month': [2, 3], + 'day': [4, 5]}) + result = pd.to_datetime(df, box=False) + expected = np.array(['2015-02-04', '2016-03-05'], + dtype='datetime64[ns]') + tm.assert_numpy_array_equal(result, expected) + + def test_dataframe_utc_true(self): + # GH 23760 + df = pd.DataFrame({'year': [2015, 2016], + 'month': [2, 3], + 'day': [4, 5]}) + result = pd.to_datetime(df, utc=True) + expected = pd.Series(np.array(['2015-02-04', '2016-03-05'], + dtype='datetime64[ns]')).dt.tz_localize('UTC') + tm.assert_series_equal(result, expected) + def test_to_datetime_errors_ignore_utc_true(self): # GH 23758 result = pd.to_datetime([1], unit='s', box=True, utc=True, From 97c05f3af88a0d3b4fa738c484083f39e4398a75 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Mon, 24 Dec 2018 00:06:46 -0800 Subject: [PATCH 3/3] fix whatsnew messages --- doc/source/whatsnew/v0.24.0.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/whatsnew/v0.24.0.rst b/doc/source/whatsnew/v0.24.0.rst index 39ed87134def1..1bc1a28ef0fe9 100644 --- a/doc/source/whatsnew/v0.24.0.rst +++ b/doc/source/whatsnew/v0.24.0.rst @@ -1318,7 +1318,7 @@ Datetimelike - Bug in :class:`DatetimeIndex` where constructing a :class:`DatetimeIndex` from a :class:`Categorical` or :class:`CategoricalIndex` would incorrectly drop timezone information (:issue:`18664`) - Bug in :class:`DatetimeIndex` and :class:`TimedeltaIndex` where indexing with ``Ellipsis`` would incorrectly lose the index's ``freq`` attribute (:issue:`21282`) - Clarified error message produced when passing an incorrect ``freq`` argument to :class:`DatetimeIndex` with ``NaT`` as the first entry in the passed data (:issue:`11587`) -- Bug in :func:`to_datetime` where ``box`` and ``utc``arguments were ignored when passing a :class:`DataFrame` or ``dict`` of unit mappings (:issue:`23760`) +- Bug in :func:`to_datetime` where ``box`` and ``utc`` arguments were ignored when passing a :class:`DataFrame` or ``dict`` of unit mappings (:issue:`23760`) Timedelta ^^^^^^^^^ @@ -1362,7 +1362,7 @@ Timezones - Bug in :class:`DatetimeIndex` constructor where ``NaT`` and ``dateutil.tz.tzlocal`` would raise an ``OutOfBoundsDatetime`` error (:issue:`23807`) - Bug in :meth:`DatetimeIndex.tz_localize` and :meth:`Timestamp.tz_localize` with ``dateutil.tz.tzlocal`` near a DST transition that would return an incorrectly localized datetime (:issue:`23807`) - Bug in :class:`Timestamp` constructor where a ``dateutil.tz.tzutc`` timezone passed with a ``datetime.datetime`` argument would be converted to a ``pytz.UTC`` timezone (:issue:`23807`) -- Bug in :func:`to_datetime` where ``utc=True`` was not respected when specifying a ``unit`` and ``errors=='ignore'`` (:issue:`23758`) +- Bug in :func:`to_datetime` where ``utc=True`` was not respected when specifying a ``unit`` and ``errors='ignore'`` (:issue:`23758`) Offsets ^^^^^^^