From 704516f5c9bbd03a27c2423414a21458ce83b9a7 Mon Sep 17 00:00:00 2001 From: maheshbapatu Date: Thu, 5 Dec 2019 16:29:02 +0000 Subject: [PATCH 1/7] Closes #30011 --- pandas/core/tools/datetimes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/tools/datetimes.py b/pandas/core/tools/datetimes.py index 453d1cca2e085..ef8a239d32a3f 100644 --- a/pandas/core/tools/datetimes.py +++ b/pandas/core/tools/datetimes.py @@ -940,7 +940,7 @@ def calc_with_mask(carg, mask): # try intlike / strings that are ints try: return calc(arg.astype(np.int64)) - except (ValueError, OverflowError): + except (ValueError, OverflowError, TypeError): pass # a float with actual np.nan From 5351805fed1407590d8e19fe55903bd3554b7729 Mon Sep 17 00:00:00 2001 From: maheshbapatu Date: Thu, 5 Dec 2019 16:29:02 +0000 Subject: [PATCH 2/7] This avoids failure to_datetime when value passed is None. This fix is for pandas versions > 0.24 Closes #30011 --- pandas/tests/indexes/datetimes/test_tools.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/pandas/tests/indexes/datetimes/test_tools.py b/pandas/tests/indexes/datetimes/test_tools.py index 4e5d624eba844..d096c398de91d 100644 --- a/pandas/tests/indexes/datetimes/test_tools.py +++ b/pandas/tests/indexes/datetimes/test_tools.py @@ -101,6 +101,25 @@ def test_to_datetime_format_YYYYMMDD(self, cache): expected = Series(["20121231", "20141231", "NaT"], dtype="M8[ns]") tm.assert_series_equal(result, expected) + @pytest.mark.parametrize( + "input_s, expected", + [ + # strings with None + [ + Series(["19801222", None, "20010112"]), + Series([Timestamp("19801222"), Timestamp(None), Timestamp("20010112")]), + ], + # Integers with None + [ + Series([19801222, None, 20010112]), + Series([Timestamp("19801222"), Timestamp(None), Timestamp("20010112")]), + ], + ], + ) + def test_to_datetime_format_YYYYMMDD_with_none(self, input_s, expected): + result = pd.to_datetime(input_s, format="%Y%m%d") + tm.assert_series_equal(result, expected) + @pytest.mark.parametrize( "input_s, expected", [ From 4d6e0527ea4b9f8c02a54d38d7a324e7da983851 Mon Sep 17 00:00:00 2001 From: maheshbapatu Date: Thu, 5 Dec 2019 16:29:02 +0000 Subject: [PATCH 3/7] This avoids failure to_datetime when value passed is None. This fix is for pandas versions > 0.24 Closes #30011 --- pandas/tests/indexes/datetimes/test_tools.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/pandas/tests/indexes/datetimes/test_tools.py b/pandas/tests/indexes/datetimes/test_tools.py index 4e5d624eba844..775e3573fc607 100644 --- a/pandas/tests/indexes/datetimes/test_tools.py +++ b/pandas/tests/indexes/datetimes/test_tools.py @@ -101,6 +101,24 @@ def test_to_datetime_format_YYYYMMDD(self, cache): expected = Series(["20121231", "20141231", "NaT"], dtype="M8[ns]") tm.assert_series_equal(result, expected) + @pytest.mark.parametrize( + "input_s", + [ + # None with Strings + Series(["19801222", None, "20010112"]), + # None with Integers + Series([19801222, None, 20010112]) + ], + ) + def test_to_datetime_format_YYYYMMDD_with_none(self,input_s): + # GH 30011 + # format='%Y%m%d' + # with None + expected = Series([Timestamp("19801222"), Timestamp(None),Timestamp("20010112")]) + + result = pd.to_datetime(input_s, format="%Y%m%d") + tm.assert_series_equal(result, expected) + @pytest.mark.parametrize( "input_s, expected", [ From 46cd6d0ea46e2d9e30960ca4a93717d15f8d5e89 Mon Sep 17 00:00:00 2001 From: maheshbapatu Date: Sun, 8 Dec 2019 10:46:08 +0000 Subject: [PATCH 4/7] This avoids failure to_datetime when value passed is None to to_datetime. issue #30011 --- pandas/tests/indexes/datetimes/test_tools.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pandas/tests/indexes/datetimes/test_tools.py b/pandas/tests/indexes/datetimes/test_tools.py index 72b41b1bbef89..f45e04b6f8ec2 100644 --- a/pandas/tests/indexes/datetimes/test_tools.py +++ b/pandas/tests/indexes/datetimes/test_tools.py @@ -107,15 +107,16 @@ def test_to_datetime_format_YYYYMMDD(self, cache): # None with Strings Series(["19801222", None, "20010112"]), # None with Integers - Series([19801222, None, 20010112]) + Series([19801222, None, 20010112]), ], ) def test_to_datetime_format_YYYYMMDD_with_none(self, input_s): # GH 30011 # format='%Y%m%d' # with None - expected = Series([Timestamp("19801222"), Timestamp(None), - Timestamp("20010112")]) + expected = Series( + [Timestamp("19801222"), Timestamp(None), Timestamp("20010112")] + ) result = pd.to_datetime(input_s, format="%Y%m%d") tm.assert_series_equal(result, expected) From c07c1cf596f4a0907db6f2fbcff57584c2671531 Mon Sep 17 00:00:00 2001 From: maheshbapatu Date: Sun, 8 Dec 2019 16:33:41 +0000 Subject: [PATCH 5/7] =?UTF-8?q?This=20avoids=20failure=20in=20to=5Fdatetim?= =?UTF-8?q?e=20when=20value=20passed=20is=20None.=20After=20changes=20it?= =?UTF-8?q?=20will=20return=20NaT=20instead=20of=20TypeError.=E2=80=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- doc/source/whatsnew/v1.0.0.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/whatsnew/v1.0.0.rst b/doc/source/whatsnew/v1.0.0.rst index 4ce4c12483b36..1afab0b29531e 100644 --- a/doc/source/whatsnew/v1.0.0.rst +++ b/doc/source/whatsnew/v1.0.0.rst @@ -652,6 +652,7 @@ Datetimelike - Bug in :meth:`Series.var` failing to raise ``TypeError`` when called with ``timedelta64[ns]`` dtype (:issue:`28289`) - Bug in :meth:`DatetimeIndex.strftime` and :meth:`Series.dt.strftime` where ``NaT`` was converted to the string ``'NaT'`` instead of ``np.nan`` (:issue:`29578`) - Bug in :attr:`Timestamp.resolution` being a property instead of a class attribute (:issue:`29910`) +- Bug in :func:`pandas.core.tools.datetimes._attempt_YYYYMMDD` to_datetime() when called with ``None`` raising ``TypeError`` instead of returning ``NaT``. After changes it will return ``NaT`` (:issue:`30011`) Timedelta ^^^^^^^^^ From 13c5c42079548c60d6045b78a7d5aee0654b1e3c Mon Sep 17 00:00:00 2001 From: maheshbapatu Date: Sun, 8 Dec 2019 21:20:58 +0000 Subject: [PATCH 6/7] This avoids failure in to_datetime when value passed is None. --- doc/source/whatsnew/v1.0.0.rst | 2 +- pandas/core/tools/datetimes.py | 4 ++-- pandas/tests/indexes/datetimes/test_tools.py | 13 ++++++++++--- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/doc/source/whatsnew/v1.0.0.rst b/doc/source/whatsnew/v1.0.0.rst index 1afab0b29531e..8caba5cf9d53f 100644 --- a/doc/source/whatsnew/v1.0.0.rst +++ b/doc/source/whatsnew/v1.0.0.rst @@ -652,7 +652,7 @@ Datetimelike - Bug in :meth:`Series.var` failing to raise ``TypeError`` when called with ``timedelta64[ns]`` dtype (:issue:`28289`) - Bug in :meth:`DatetimeIndex.strftime` and :meth:`Series.dt.strftime` where ``NaT`` was converted to the string ``'NaT'`` instead of ``np.nan`` (:issue:`29578`) - Bug in :attr:`Timestamp.resolution` being a property instead of a class attribute (:issue:`29910`) -- Bug in :func:`pandas.core.tools.datetimes._attempt_YYYYMMDD` to_datetime() when called with ``None`` raising ``TypeError`` instead of returning ``NaT``. After changes it will return ``NaT`` (:issue:`30011`) +- Bug in :func:`pandas.to_datetime` when called with ``None`` raising ``TypeError`` instead of returning ``NaT`` (:issue:`30011`) Timedelta ^^^^^^^^^ diff --git a/pandas/core/tools/datetimes.py b/pandas/core/tools/datetimes.py index ef8a239d32a3f..f5695343282a6 100644 --- a/pandas/core/tools/datetimes.py +++ b/pandas/core/tools/datetimes.py @@ -947,14 +947,14 @@ def calc_with_mask(carg, mask): try: carg = arg.astype(np.float64) return calc_with_mask(carg, notna(carg)) - except (ValueError, OverflowError): + except (ValueError, OverflowError, TypeError): pass # string with NaN-like try: mask = ~algorithms.isin(arg, list(tslib.nat_strings)) return calc_with_mask(arg, mask) - except (ValueError, OverflowError): + except (ValueError, OverflowError, TypeError): pass return None diff --git a/pandas/tests/indexes/datetimes/test_tools.py b/pandas/tests/indexes/datetimes/test_tools.py index f45e04b6f8ec2..5351da8d1d84c 100644 --- a/pandas/tests/indexes/datetimes/test_tools.py +++ b/pandas/tests/indexes/datetimes/test_tools.py @@ -105,9 +105,9 @@ def test_to_datetime_format_YYYYMMDD(self, cache): "input_s", [ # None with Strings - Series(["19801222", None, "20010112"]), + Series(["19801222", None, "20010112", np.nan, "NaT", pd.NaT]), # None with Integers - Series([19801222, None, 20010112]), + Series([19801222, None, 20010112, np.nan, "NaT", pd.NaT]), ], ) def test_to_datetime_format_YYYYMMDD_with_none(self, input_s): @@ -115,7 +115,14 @@ def test_to_datetime_format_YYYYMMDD_with_none(self, input_s): # format='%Y%m%d' # with None expected = Series( - [Timestamp("19801222"), Timestamp(None), Timestamp("20010112")] + [ + Timestamp("19801222"), + Timestamp(None), + Timestamp("20010112"), + Timestamp(np.nan), + Timestamp("NaT"), + Timestamp(pd.NaT), + ] ) result = pd.to_datetime(input_s, format="%Y%m%d") tm.assert_series_equal(result, expected) From a47da79d58751134626f6423299d944455ce9185 Mon Sep 17 00:00:00 2001 From: maheshbapatu Date: Tue, 10 Dec 2019 19:38:58 +0000 Subject: [PATCH 7/7] This avoids failure in to_datetime when value passed is Null type. --- pandas/tests/indexes/datetimes/test_tools.py | 27 +++++++++----------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/pandas/tests/indexes/datetimes/test_tools.py b/pandas/tests/indexes/datetimes/test_tools.py index 5351da8d1d84c..ee4611d0f56df 100644 --- a/pandas/tests/indexes/datetimes/test_tools.py +++ b/pandas/tests/indexes/datetimes/test_tools.py @@ -104,27 +104,24 @@ def test_to_datetime_format_YYYYMMDD(self, cache): @pytest.mark.parametrize( "input_s", [ - # None with Strings - Series(["19801222", None, "20010112", np.nan, "NaT", pd.NaT]), - # None with Integers - Series([19801222, None, 20010112, np.nan, "NaT", pd.NaT]), + # Null values with Strings + ["19801222", "20010112", None], + ["19801222", "20010112", np.nan], + ["19801222", "20010112", pd.NaT], + ["19801222", "20010112", "NaT"], + # Null values with Integers + [19801222, 20010112, None], + [19801222, 20010112, np.nan], + [19801222, 20010112, pd.NaT], + [19801222, 20010112, "NaT"], ], ) def test_to_datetime_format_YYYYMMDD_with_none(self, input_s): # GH 30011 # format='%Y%m%d' # with None - expected = Series( - [ - Timestamp("19801222"), - Timestamp(None), - Timestamp("20010112"), - Timestamp(np.nan), - Timestamp("NaT"), - Timestamp(pd.NaT), - ] - ) - result = pd.to_datetime(input_s, format="%Y%m%d") + expected = Series([Timestamp("19801222"), Timestamp("20010112"), pd.NaT]) + result = Series(pd.to_datetime(input_s, format="%Y%m%d")) tm.assert_series_equal(result, expected) @pytest.mark.parametrize(