From 4e3129832514696d354c4fdb2a37c447426a433c Mon Sep 17 00:00:00 2001 From: Brock Date: Mon, 18 Jan 2021 09:43:08 -0800 Subject: [PATCH 1/6] DEPR: astype dt64<->dt64tz --- pandas/core/dtypes/cast.py | 55 +++++++++++++++++++ pandas/core/dtypes/dtypes.py | 2 +- pandas/core/internals/blocks.py | 3 + pandas/tests/arrays/test_datetimes.py | 15 +++-- pandas/tests/frame/methods/test_astype.py | 4 +- .../indexes/datetimes/methods/test_astype.py | 12 +++- pandas/tests/indexes/test_base.py | 9 ++- pandas/tests/indexes/test_common.py | 7 +++ pandas/tests/series/methods/test_astype.py | 8 ++- .../series/methods/test_convert_dtypes.py | 24 +++++++- 10 files changed, 124 insertions(+), 15 deletions(-) diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index 0941967ef6bee..d6797cc5edd25 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -942,6 +942,32 @@ def coerce_indexer_dtype(indexer, categories): return ensure_int64(indexer) +def _find_level() -> int: + """ + Find the appropriate stacklevel with which to issue a warning for astype. + """ + import inspect + + stack = inspect.stack() + + # find the lowest-level "astype" call that got us here + for n in range(2, 6): + if stack[n].function == "astype": + break + + while stack[n].function in ["astype", "apply", "_astype"]: + # e.g. + # bump up Block.astype -> BlockManager.astype -> NDFrame.astype + # bump up Datetime.Array.astype -> DatetimeIndex.astype + n += 1 + + if stack[n].function == "__init__": + # Series.__init__ + n += 1 + + return n + + def astype_dt64_to_dt64tz( values: ArrayLike, dtype: DtypeObj, copy: bool, via_utc: bool = False ) -> DatetimeArray: @@ -964,6 +990,16 @@ def astype_dt64_to_dt64tz( if copy: # this should be the only copy values = values.copy() + + level = _find_level() + warnings.warn( + "Using .astype to convert from timezone-naive dtype to " + "timezone-aware dtype is deprecated and will raise in a " + "future version. Use ser.dt.tz_localize instead.", + FutureWarning, + stacklevel=level, + ) + # FIXME: GH#33401 this doesn't match DatetimeArray.astype, which # goes through the `not via_utc` path return values.tz_localize("UTC").tz_convert(dtype.tz) @@ -973,6 +1009,15 @@ def astype_dt64_to_dt64tz( if values.tz is None and aware: dtype = cast(DatetimeTZDtype, dtype) + level = _find_level() + warnings.warn( + "Using .astype to convert from timezone-naive dtype to " + "timezone-aware dtype is deprecated and will raise in a " + "future version. Use obj.tz_localize instead.", + FutureWarning, + stacklevel=level, + ) + return values.tz_localize(dtype.tz) elif aware: @@ -984,6 +1029,16 @@ def astype_dt64_to_dt64tz( return result elif values.tz is not None: + level = _find_level() + warnings.warn( + "Using .astype to convert from timezone-aware dtype to " + "timezone-naive dtype is deprecated and will raise in a " + "future version. Use obj.tz_localize(None) or " + "obj.tz_convert('UTC').tz_localize(None) instead", + FutureWarning, + stacklevel=level, + ) + result = values.tz_convert("UTC").tz_localize(None) if copy: result = result.copy() diff --git a/pandas/core/dtypes/dtypes.py b/pandas/core/dtypes/dtypes.py index cefab33976ba8..a3983bb630c75 100644 --- a/pandas/core/dtypes/dtypes.py +++ b/pandas/core/dtypes/dtypes.py @@ -432,7 +432,7 @@ def _hash_categories(categories, ordered: Ordered = True) -> int: if DatetimeTZDtype.is_dtype(categories.dtype): # Avoid future warning. - categories = categories.astype("datetime64[ns]") + categories = categories.view("datetime64[ns]") cat_array = hash_array(np.asarray(categories), categorize=False) if ordered: diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index 1356b9d3b2ca3..967e23133dbcb 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -2183,6 +2183,9 @@ def get_values(self, dtype: Optional[Dtype] = None): def external_values(self): # NB: this is different from np.asarray(self.values), since that # return an object-dtype ndarray of Timestamps. + if self.is_datetimetz: + # avoid FutureWarning in .astype in casting from dt64t to dt64 + return self.values._data return np.asarray(self.values.astype("datetime64[ns]", copy=False)) def fillna(self, value, limit=None, inplace=False, downcast=None): diff --git a/pandas/tests/arrays/test_datetimes.py b/pandas/tests/arrays/test_datetimes.py index 52f71f8c8f505..ea44e5d477fc6 100644 --- a/pandas/tests/arrays/test_datetimes.py +++ b/pandas/tests/arrays/test_datetimes.py @@ -175,11 +175,18 @@ def test_astype_to_same(self): ) def test_astype_copies(self, dtype, other): # https://github.com/pandas-dev/pandas/pull/32490 - s = pd.Series([1, 2], dtype=dtype) - orig = s.copy() - t = s.astype(other) + ser = pd.Series([1, 2], dtype=dtype) + orig = ser.copy() + + warn = None + if (dtype == "datetime64[ns]") ^ (other == "datetime64[ns]"): + # deprecated in favor of tz_localize + warn = FutureWarning + + with tm.assert_produces_warning(warn): + t = ser.astype(other) t[:] = pd.NaT - tm.assert_series_equal(s, orig) + tm.assert_series_equal(ser, orig) @pytest.mark.parametrize("dtype", [int, np.int32, np.int64, "uint32", "uint64"]) def test_astype_int(self, dtype): diff --git a/pandas/tests/frame/methods/test_astype.py b/pandas/tests/frame/methods/test_astype.py index a4da77548b920..df98c78e78fb6 100644 --- a/pandas/tests/frame/methods/test_astype.py +++ b/pandas/tests/frame/methods/test_astype.py @@ -515,7 +515,9 @@ def test_astype_dt64tz(self, timezone_frame): result = timezone_frame.astype(object) tm.assert_frame_equal(result, expected) - result = timezone_frame.astype("datetime64[ns]") + with tm.assert_produces_warning(FutureWarning): + # dt64tz->dt64 deprecated + result = timezone_frame.astype("datetime64[ns]") expected = DataFrame( { "A": date_range("20130101", periods=3), diff --git a/pandas/tests/indexes/datetimes/methods/test_astype.py b/pandas/tests/indexes/datetimes/methods/test_astype.py index 98d5e074091de..11e0552c671bc 100644 --- a/pandas/tests/indexes/datetimes/methods/test_astype.py +++ b/pandas/tests/indexes/datetimes/methods/test_astype.py @@ -58,7 +58,9 @@ def test_astype_with_tz(self): # with tz rng = date_range("1/1/2000", periods=10, tz="US/Eastern") - result = rng.astype("datetime64[ns]") + with tm.assert_produces_warning(FutureWarning): + # deprecated + result = rng.astype("datetime64[ns]") expected = ( date_range("1/1/2000", periods=10, tz="US/Eastern") .tz_convert("UTC") @@ -78,7 +80,9 @@ def test_astype_tznaive_to_tzaware(self): # GH 18951: tz-naive to tz-aware idx = date_range("20170101", periods=4) idx = idx._with_freq(None) # tz_localize does not preserve freq - result = idx.astype("datetime64[ns, US/Eastern]") + with tm.assert_produces_warning(FutureWarning): + # dt64->dt64tz deprecated + result = idx.astype("datetime64[ns, US/Eastern]") expected = date_range("20170101", periods=4, tz="US/Eastern") expected = expected._with_freq(None) tm.assert_index_equal(result, expected) @@ -155,7 +159,9 @@ def test_astype_datetime64(self): assert result is idx idx_tz = DatetimeIndex(["2016-05-16", "NaT", NaT, np.NaN], tz="EST", name="idx") - result = idx_tz.astype("datetime64[ns]") + with tm.assert_produces_warning(FutureWarning): + # dt64tz->dt64 deprecated + result = idx_tz.astype("datetime64[ns]") expected = DatetimeIndex( ["2016-05-16 05:00:00", "NaT", "NaT", "NaT"], dtype="datetime64[ns]", diff --git a/pandas/tests/indexes/test_base.py b/pandas/tests/indexes/test_base.py index 1eca7f7a5d261..383584a6cd787 100644 --- a/pandas/tests/indexes/test_base.py +++ b/pandas/tests/indexes/test_base.py @@ -349,6 +349,9 @@ def test_constructor_dtypes_datetime(self, tz_naive_fixture, attr, klass): index = index.tz_localize(tz_naive_fixture) dtype = index.dtype + warn = None if tz_naive_fixture is None else FutureWarning + # astype dt64 -> dt64tz deprecated + if attr == "asi8": result = DatetimeIndex(arg).tz_localize(tz_naive_fixture) else: @@ -356,7 +359,8 @@ def test_constructor_dtypes_datetime(self, tz_naive_fixture, attr, klass): tm.assert_index_equal(result, index) if attr == "asi8": - result = DatetimeIndex(arg).astype(dtype) + with tm.assert_produces_warning(warn): + result = DatetimeIndex(arg).astype(dtype) else: result = klass(arg, dtype=dtype) tm.assert_index_equal(result, index) @@ -368,7 +372,8 @@ def test_constructor_dtypes_datetime(self, tz_naive_fixture, attr, klass): tm.assert_index_equal(result, index) if attr == "asi8": - result = DatetimeIndex(list(arg)).astype(dtype) + with tm.assert_produces_warning(warn): + result = DatetimeIndex(list(arg)).astype(dtype) else: result = klass(list(arg), dtype=dtype) tm.assert_index_equal(result, index) diff --git a/pandas/tests/indexes/test_common.py b/pandas/tests/indexes/test_common.py index 24244952c6545..4ee1ba24df1d4 100644 --- a/pandas/tests/indexes/test_common.py +++ b/pandas/tests/indexes/test_common.py @@ -352,6 +352,13 @@ def test_astype_preserves_name(self, index, dtype): if dtype in ["int64", "uint64"]: if needs_i8_conversion(index.dtype): warn = FutureWarning + elif ( + isinstance(index, DatetimeIndex) + and index.tz is not None + and dtype == "datetime64[ns]" + ): + # This astype is deprecated in favor of tz_localize + warn = FutureWarning try: # Some of these conversions cannot succeed so we use a try / except with tm.assert_produces_warning(warn, check_stacklevel=False): diff --git a/pandas/tests/series/methods/test_astype.py b/pandas/tests/series/methods/test_astype.py index 3cd9d52f8e754..1a141e3201d57 100644 --- a/pandas/tests/series/methods/test_astype.py +++ b/pandas/tests/series/methods/test_astype.py @@ -193,10 +193,14 @@ def test_astype_datetime64tz(self): tm.assert_series_equal(result, expected) # astype - datetime64[ns, tz] - result = Series(s.values).astype("datetime64[ns, US/Eastern]") + with tm.assert_produces_warning(FutureWarning): + # dt64->dt64tz astype deprecated + result = Series(s.values).astype("datetime64[ns, US/Eastern]") tm.assert_series_equal(result, s) - result = Series(s.values).astype(s.dtype) + with tm.assert_produces_warning(FutureWarning): + # dt64->dt64tz astype deprecated + result = Series(s.values).astype(s.dtype) tm.assert_series_equal(result, s) result = s.astype("datetime64[ns, CET]") diff --git a/pandas/tests/series/methods/test_convert_dtypes.py b/pandas/tests/series/methods/test_convert_dtypes.py index 8c1a674b705d5..f7b49c187c794 100644 --- a/pandas/tests/series/methods/test_convert_dtypes.py +++ b/pandas/tests/series/methods/test_convert_dtypes.py @@ -156,8 +156,18 @@ class TestSeriesConvertDtypes: def test_convert_dtypes( self, data, maindtype, params, expected_default, expected_other ): + warn = None + if ( + hasattr(data, "dtype") + and data.dtype == "M8[ns]" + and isinstance(maindtype, pd.DatetimeTZDtype) + ): + # this astype is deprecated in favor of tz_localize + warn = FutureWarning + if maindtype is not None: - series = pd.Series(data, dtype=maindtype) + with tm.assert_produces_warning(warn): + series = pd.Series(data, dtype=maindtype) else: series = pd.Series(data) @@ -177,7 +187,17 @@ def test_convert_dtypes( if all(params_dict[key] is val for key, val in zip(spec[::2], spec[1::2])): expected_dtype = dtype - expected = pd.Series(data, dtype=expected_dtype) + warn2 = None + if ( + hasattr(data, "dtype") + and data.dtype == "M8[ns]" + and isinstance(expected_dtype, pd.DatetimeTZDtype) + ): + # this astype is deprecated in favor of tz_localize + warn2 = FutureWarning + + with tm.assert_produces_warning(warn2): + expected = pd.Series(data, dtype=expected_dtype) tm.assert_series_equal(result, expected) # Test that it is a copy From c555220ea83465c1a1b45e4807d278e6abc8f56c Mon Sep 17 00:00:00 2001 From: Brock Date: Mon, 18 Jan 2021 09:48:44 -0800 Subject: [PATCH 2/6] Whatnsew --- doc/source/whatsnew/v1.3.0.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/whatsnew/v1.3.0.rst b/doc/source/whatsnew/v1.3.0.rst index 6a85bfd852e19..854af05427ba2 100644 --- a/doc/source/whatsnew/v1.3.0.rst +++ b/doc/source/whatsnew/v1.3.0.rst @@ -192,6 +192,7 @@ Deprecations - Deprecated comparison of :class:`Timestamp` object with ``datetime.date`` objects. Instead of e.g. ``ts <= mydate`` use ``ts <= pd.Timestamp(mydate)`` or ``ts.date() <= mydate`` (:issue:`36131`) - Deprecated :attr:`Rolling.win_type` returning ``"freq"`` (:issue:`38963`) - Deprecated :attr:`Rolling.is_datetimelike` (:issue:`38963`) +- Using ``.astype`` to convert between ``datetime64[ns]`` dtype and :cls:`DatetimeTZDtype` is deprecated and will raise in a future version, use ``obj.tz_localize`` or ``obj.dt.tz_localize`` instead (:issue:`38622`) - .. --------------------------------------------------------------------------- From 47a3a95bc99d9c02b0cd6bcece3b830572de8dc2 Mon Sep 17 00:00:00 2001 From: Brock Date: Mon, 18 Jan 2021 21:39:12 -0800 Subject: [PATCH 3/6] update doc --- doc/source/user_guide/timeseries.rst | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/doc/source/user_guide/timeseries.rst b/doc/source/user_guide/timeseries.rst index 8909f5b33066b..01ff62d984544 100644 --- a/doc/source/user_guide/timeseries.rst +++ b/doc/source/user_guide/timeseries.rst @@ -2605,17 +2605,10 @@ For example, to localize and convert a naive stamp to time zone aware. s_naive.dt.tz_localize("UTC").dt.tz_convert("US/Eastern") Time zone information can also be manipulated using the ``astype`` method. -This method can localize and convert time zone naive timestamps or -convert time zone aware timestamps. +This method can convert between different timezone-aware dtypes. .. ipython:: python - # localize and convert a naive time zone - s_naive.astype("datetime64[ns, US/Eastern]") - - # make an aware tz naive - s_aware.astype("datetime64[ns]") - # convert to a new time zone s_aware.astype("datetime64[ns, CET]") From b162a1b11793ff3ffe63a3a34d6df2ba4c6d1d5a Mon Sep 17 00:00:00 2001 From: Brock Date: Wed, 20 Jan 2021 09:08:51 -0800 Subject: [PATCH 4/6] typo fixup --- doc/source/whatsnew/v1.3.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v1.3.0.rst b/doc/source/whatsnew/v1.3.0.rst index 41047ea2557c4..bac8f08857880 100644 --- a/doc/source/whatsnew/v1.3.0.rst +++ b/doc/source/whatsnew/v1.3.0.rst @@ -193,7 +193,7 @@ Deprecations - Deprecated :attr:`Rolling.win_type` returning ``"freq"`` (:issue:`38963`) - Deprecated :attr:`Rolling.is_datetimelike` (:issue:`38963`) - Deprecated :meth:`core.window.ewm.ExponentialMovingWindow.vol` (:issue:`39220`) -- Using ``.astype`` to convert between ``datetime64[ns]`` dtype and :cls:`DatetimeTZDtype` is deprecated and will raise in a future version, use ``obj.tz_localize`` or ``obj.dt.tz_localize`` instead (:issue:`38622`) +- Using ``.astype`` to convert between ``datetime64[ns]`` dtype and :class:`DatetimeTZDtype` is deprecated and will raise in a future version, use ``obj.tz_localize`` or ``obj.dt.tz_localize`` instead (:issue:`38622`) - .. --------------------------------------------------------------------------- From 02a4a43180bf21126814fd48506850ae78b6e9cc Mon Sep 17 00:00:00 2001 From: Brock Date: Wed, 20 Jan 2021 11:19:42 -0800 Subject: [PATCH 5/6] rename, add test for DTA --- pandas/core/dtypes/cast.py | 11 +++++------ pandas/tests/indexes/datetimes/methods/test_astype.py | 8 ++++++++ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index 94a7dafde492a..0c578004de94d 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -6,6 +6,7 @@ from contextlib import suppress from datetime import datetime, timedelta +import inspect from typing import ( TYPE_CHECKING, Any, @@ -942,12 +943,10 @@ def coerce_indexer_dtype(indexer, categories): return ensure_int64(indexer) -def _find_level() -> int: +def find_stack_level() -> int: """ Find the appropriate stacklevel with which to issue a warning for astype. """ - import inspect - stack = inspect.stack() # find the lowest-level "astype" call that got us here @@ -991,7 +990,7 @@ def astype_dt64_to_dt64tz( # this should be the only copy values = values.copy() - level = _find_level() + level = find_stack_level() warnings.warn( "Using .astype to convert from timezone-naive dtype to " "timezone-aware dtype is deprecated and will raise in a " @@ -1009,7 +1008,7 @@ def astype_dt64_to_dt64tz( if values.tz is None and aware: dtype = cast(DatetimeTZDtype, dtype) - level = _find_level() + level = find_stack_level() warnings.warn( "Using .astype to convert from timezone-naive dtype to " "timezone-aware dtype is deprecated and will raise in a " @@ -1029,7 +1028,7 @@ def astype_dt64_to_dt64tz( return result elif values.tz is not None: - level = _find_level() + level = find_stack_level() warnings.warn( "Using .astype to convert from timezone-aware dtype to " "timezone-naive dtype is deprecated and will raise in a " diff --git a/pandas/tests/indexes/datetimes/methods/test_astype.py b/pandas/tests/indexes/datetimes/methods/test_astype.py index 11e0552c671bc..bed7cb9b54eba 100644 --- a/pandas/tests/indexes/datetimes/methods/test_astype.py +++ b/pandas/tests/indexes/datetimes/methods/test_astype.py @@ -61,6 +61,10 @@ def test_astype_with_tz(self): with tm.assert_produces_warning(FutureWarning): # deprecated result = rng.astype("datetime64[ns]") + with tm.assert_produces_warning(FutureWarning): + # check DatetimeArray while we're here deprecated + rng._data.astype("datetime64[ns]") + expected = ( date_range("1/1/2000", periods=10, tz="US/Eastern") .tz_convert("UTC") @@ -83,6 +87,10 @@ def test_astype_tznaive_to_tzaware(self): with tm.assert_produces_warning(FutureWarning): # dt64->dt64tz deprecated result = idx.astype("datetime64[ns, US/Eastern]") + with tm.assert_produces_warning(FutureWarning): + # dt64->dt64tz deprecated + idx._data.astype("datetime64[ns, US/Eastern]") + expected = date_range("20170101", periods=4, tz="US/Eastern") expected = expected._with_freq(None) tm.assert_index_equal(result, expected) From 0e4c3caf0dfee14eee192e339411327790f999e7 Mon Sep 17 00:00:00 2001 From: Brock Date: Wed, 20 Jan 2021 14:30:11 -0800 Subject: [PATCH 6/6] REF: move find_stack_level to util._exceptions --- pandas/core/dtypes/cast.py | 26 +------------------------- pandas/util/_exceptions.py | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 25 deletions(-) diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index 0c578004de94d..5d53f6add251a 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -6,7 +6,6 @@ from contextlib import suppress from datetime import datetime, timedelta -import inspect from typing import ( TYPE_CHECKING, Any, @@ -39,6 +38,7 @@ tz_compare, ) from pandas._typing import AnyArrayLike, ArrayLike, Dtype, DtypeObj, Scalar +from pandas.util._exceptions import find_stack_level from pandas.util._validators import validate_bool_kwarg from pandas.core.dtypes.common import ( @@ -943,30 +943,6 @@ def coerce_indexer_dtype(indexer, categories): return ensure_int64(indexer) -def find_stack_level() -> int: - """ - Find the appropriate stacklevel with which to issue a warning for astype. - """ - stack = inspect.stack() - - # find the lowest-level "astype" call that got us here - for n in range(2, 6): - if stack[n].function == "astype": - break - - while stack[n].function in ["astype", "apply", "_astype"]: - # e.g. - # bump up Block.astype -> BlockManager.astype -> NDFrame.astype - # bump up Datetime.Array.astype -> DatetimeIndex.astype - n += 1 - - if stack[n].function == "__init__": - # Series.__init__ - n += 1 - - return n - - def astype_dt64_to_dt64tz( values: ArrayLike, dtype: DtypeObj, copy: bool, via_utc: bool = False ) -> DatetimeArray: diff --git a/pandas/util/_exceptions.py b/pandas/util/_exceptions.py index 54bfcbddfc3dd..5ca96a1f9989f 100644 --- a/pandas/util/_exceptions.py +++ b/pandas/util/_exceptions.py @@ -1,4 +1,5 @@ import contextlib +import inspect from typing import Tuple @@ -17,3 +18,27 @@ def rewrite_exception(old_name: str, new_name: str): args = args + err.args[1:] err.args = args raise + + +def find_stack_level() -> int: + """ + Find the appropriate stacklevel with which to issue a warning for astype. + """ + stack = inspect.stack() + + # find the lowest-level "astype" call that got us here + for n in range(2, 6): + if stack[n].function == "astype": + break + + while stack[n].function in ["astype", "apply", "_astype"]: + # e.g. + # bump up Block.astype -> BlockManager.astype -> NDFrame.astype + # bump up Datetime.Array.astype -> DatetimeIndex.astype + n += 1 + + if stack[n].function == "__init__": + # Series.__init__ + n += 1 + + return n