From e48023bfc111f1786bc712e81ef8bca9ed37fc3c Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Mon, 31 Dec 2018 10:59:21 -0800 Subject: [PATCH 1/6] make eadata the attribute and data the property --- pandas/core/arrays/datetimelike.py | 4 +- pandas/core/arrays/datetimes.py | 22 ++++--- pandas/core/indexes/datetimes.py | 98 ++++++++++++++++++------------ pandas/core/indexes/timedeltas.py | 40 ++++++------ 4 files changed, 100 insertions(+), 64 deletions(-) diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index 6b7199c019c48..a37fa614a65ba 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -1138,8 +1138,8 @@ def _time_shift(self, periods, freq=None): freq = frequencies.to_offset(freq) offset = periods * freq result = self + offset - if hasattr(self, 'tz'): - result._tz = self.tz + if getattr(self, 'tz', None): + result._dtype = self._dtype return result if periods == 0: diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index 197eeed5b5ddc..0b38d34a2ab33 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -210,7 +210,7 @@ class DatetimeArrayMixin(dtl.DatetimeLikeArrayMixin, # Constructors _attributes = ["freq", "tz"] - _tz = None + _dtype = None _freq = None @classmethod @@ -231,8 +231,13 @@ def _simple_new(cls, values, freq=None, tz=None): result = object.__new__(cls) result._data = values result._freq = freq - tz = timezones.maybe_get_tz(tz) - result._tz = timezones.tz_standardize(tz) + if tz is None: + dtype = _NS_DTYPE + else: + tz = timezones.maybe_get_tz(tz) + tz = timezones.tz_standardize(tz) + dtype = DatetimeTZDtype('ns', tz) + result._dtype = dtype return result def __new__(cls, values, freq=None, tz=None, dtype=None, copy=False, @@ -399,9 +404,12 @@ def dtype(self): If the values are tz-aware, then the ``DatetimeTZDtype`` is returned. """ - if self.tz is None: - return _NS_DTYPE - return DatetimeTZDtype('ns', self.tz) + return self._dtype + + @property + def _tz(self): + # TODO: Can we get rid of the private version of this? + return getattr(self._dtype, "tz", None) @property def tz(self): @@ -411,7 +419,7 @@ def tz(self): Returns ------- datetime.tzinfo, pytz.tzinfo.BaseTZInfo, dateutil.tz.tz.tzfile, or None - Returns None when the array is tz-naive. + Returns None when the array is tz-naive. """ # GH 18595 return self._tz diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py index ffee263c0bedc..d5c11ad364743 100644 --- a/pandas/core/indexes/datetimes.py +++ b/pandas/core/indexes/datetimes.py @@ -321,11 +321,10 @@ def _simple_new(cls, values, name=None, freq=None, tz=None, dtype=None): dtarr = DatetimeArray._simple_new(values, freq=freq, tz=tz) result = object.__new__(cls) - result._data = dtarr._data - result._freq = dtarr.freq - result._tz = dtarr.tz + result._eadata = dtarr result.name = name # For groupby perf. See note in indexes/base about _index_data + # TODO: make sure this is updated correctly if edited result._index_data = result._data result._reset_identity() return result @@ -345,19 +344,6 @@ def _values(self): else: return self.values - @property - def tz(self): - # GH 18595 - return self._tz - - @tz.setter - def tz(self, value): - # GH 3746: Prevent localizing or converting the index by setting tz - raise AttributeError("Cannot directly set timezone. Use tz_localize() " - "or tz_convert() as appropriate") - - tzinfo = tz - @property def size(self): # TODO: Remove this when we have a DatetimeTZArray @@ -416,15 +402,18 @@ def __setstate__(self, state): data = np.empty(nd_state[1], dtype=nd_state[2]) np.ndarray.__setstate__(data, nd_state) + freq = own_state[1] + tz = timezones.tz_standardize(own_state[2]) + dtarr = DatetimeArray._simple_new(data, freq=freq, tz=tz) + self.name = own_state[0] - self._freq = own_state[1] - self._tz = timezones.tz_standardize(own_state[2]) else: # pragma: no cover data = np.empty(state) np.ndarray.__setstate__(data, state) + dtarr = DatetimeArray(data) - self._data = data + self._eadata = dtarr self._reset_identity() else: @@ -502,7 +491,14 @@ def union(self, other): else: result = Index.union(this, other) if isinstance(result, DatetimeIndex): - result._tz = timezones.tz_standardize(this.tz) + if this.tz is not None: + # TODO: can this be simplified? It used to just be + # `result._tz = timezones.tz_standardize(this.tz)` + tz = timezones.tz_standardize(this.tz) + if result.tz is None: + result = result.tz_localize(tz) + else: + result = result.tz_convert(tz) if (result.freq is None and (this.freq is not None or other.freq is not None)): result.freq = to_offset(result.inferred_freq) @@ -532,9 +528,13 @@ def union_many(self, others): else: tz = this.tz this = Index.union(this, other) - if isinstance(this, DatetimeIndex): - this._tz = timezones.tz_standardize(tz) - + if isinstance(this, DatetimeIndex) and tz is not None: + # TODO: can this be simplified? it used to just be + # `this._tz = tz_standardize(tz)` + if this.tz is None: + this = this.tz_localize(timezones.tz_standardize(tz)) + else: + this = this.tz_convert(timezones.tz_standardize(tz)) return this def _can_fast_union(self, other): @@ -1129,9 +1129,43 @@ def slice_indexer(self, start=None, end=None, step=None, kind=None): # Wrapping DatetimeArray @property - def _eadata(self): - return DatetimeArray._simple_new(self._data, - tz=self.tz, freq=self.freq) + def _data(self): + return self._eadata._data + + @property + def _freq(self): + return self._eadata._freq + + @_freq.setter + def _freq(self, value): + # Validation will be handled by _eadata setter + self._eadata._freq = value + + @property + def freq(self): + return self._freq + + @freq.setter + def freq(self, value): + # validation is handled by _eadata setter + self._eadata.freq = value + + @property + def _tz(self): + return self._eadata._tz + + @property + def tz(self): + # GH#18595 + return self._tz + + @tz.setter + def tz(self, value): + # GH 3746: Prevent localizing or converting the index by setting tz + raise AttributeError("Cannot directly set timezone. Use tz_localize() " + "or tz_convert() as appropriate") + + tzinfo = tz # Compat for frequency inference, see GH#23789 _is_monotonic_increasing = Index.is_monotonic_increasing @@ -1168,18 +1202,6 @@ def offset(self, value): warnings.warn(msg, FutureWarning, stacklevel=2) self.freq = value - @property - def freq(self): - return self._freq - - @freq.setter - def freq(self, value): - if value is not None: - # let DatetimeArray to validation - self._eadata.freq = value - - self._freq = to_offset(value) - def __getitem__(self, key): result = self._eadata.__getitem__(key) if is_scalar(result): diff --git a/pandas/core/indexes/timedeltas.py b/pandas/core/indexes/timedeltas.py index 53cd358e2f906..558a466297e2a 100644 --- a/pandas/core/indexes/timedeltas.py +++ b/pandas/core/indexes/timedeltas.py @@ -235,11 +235,12 @@ def _simple_new(cls, values, name=None, freq=None, dtype=_TD_DTYPE): freq = to_offset(freq) tdarr = TimedeltaArray._simple_new(values, freq=freq) result = object.__new__(cls) - result._data = tdarr._data - result._freq = tdarr._freq + result._eadata = tdarr result.name = name # For groupby perf. See note in indexes/base about _index_data - result._index_data = result._data + # TODO: make sure this is updated correctly if edited + result._index_data = tdarr._data + result._reset_identity() return result @@ -279,8 +280,25 @@ def _format_native_types(self, na_rep='NaT', date_format=None, **kwargs): # Wrapping TimedeltaArray @property - def _eadata(self): - return TimedeltaArray._simple_new(self._data, freq=self.freq) + def _data(self): + return self._eadata._data + + @property + def _freq(self): + return self._eadata._freq + + @_freq.setter + def _freq(self, value): + self._eadata._freq = value + + @property + def freq(self): + return self._freq + + @freq.setter + def freq(self, value): + # Validation will be done in _eadata setter + self._eadata.freq = value __mul__ = _make_wrapped_arith_op("__mul__") __rmul__ = _make_wrapped_arith_op("__rmul__") @@ -316,18 +334,6 @@ def __getitem__(self, key): return result return type(self)(result, name=self.name) - @property - def freq(self): # TODO: get via eadata - return self._freq - - @freq.setter - def freq(self, value): # TODO: get via eadata - if value is not None: - # dispatch to TimedeltaArray to validate frequency - self._eadata.freq = value - - self._freq = to_offset(value) - # ------------------------------------------------------------------- @Appender(_index_shared_docs['astype']) From 288255c59b5ff74a6b0481ce1adff461d3f83811 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Mon, 31 Dec 2018 11:03:27 -0800 Subject: [PATCH 2/6] move offset prop back where it belongs --- pandas/core/indexes/datetimelike.py | 24 ++++++++++++++++++++++ pandas/core/indexes/datetimes.py | 22 -------------------- pandas/core/indexes/period.py | 31 +---------------------------- 3 files changed, 25 insertions(+), 52 deletions(-) diff --git a/pandas/core/indexes/datetimelike.py b/pandas/core/indexes/datetimelike.py index 3b6e10de1f4ff..af4df37d50789 100644 --- a/pandas/core/indexes/datetimelike.py +++ b/pandas/core/indexes/datetimelike.py @@ -100,6 +100,30 @@ def wrapper(self, other): # ------------------------------------------------------------------------ + @property + def offset(self): + """ + get/set the frequency of the instance + """ + msg = ('{cls}.offset has been deprecated and will be removed ' + 'in a future version; use {cls}.freq instead.' + .format(cls=type(self).__name__)) + warnings.warn(msg, FutureWarning, stacklevel=2) + return self.freq + + @offset.setter + def offset(self, value): + """ + get/set the frequency of the instance + """ + msg = ('{cls}.offset has been deprecated and will be removed ' + 'in a future version; use {cls}.freq instead.' + .format(cls=type(self).__name__)) + warnings.warn(msg, FutureWarning, stacklevel=2) + self.freq = value + + # ------------------------------------------------------------------------ + def equals(self, other): """ Determines if two Index objects contain the same elements. diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py index d5c11ad364743..2be47b5b11461 100644 --- a/pandas/core/indexes/datetimes.py +++ b/pandas/core/indexes/datetimes.py @@ -1180,28 +1180,6 @@ def tz(self, value): _has_same_tz = ea_passthrough("_has_same_tz") __array__ = ea_passthrough("__array__") - @property - def offset(self): - """ - get/set the frequency of the instance - """ - msg = ('{cls}.offset has been deprecated and will be removed ' - 'in a future version; use {cls}.freq instead.' - .format(cls=type(self).__name__)) - warnings.warn(msg, FutureWarning, stacklevel=2) - return self.freq - - @offset.setter - def offset(self, value): - """ - get/set the frequency of the instance - """ - msg = ('{cls}.offset has been deprecated and will be removed ' - 'in a future version; use {cls}.freq instead.' - .format(cls=type(self).__name__)) - warnings.warn(msg, FutureWarning, stacklevel=2) - self.freq = value - def __getitem__(self, key): result = self._eadata.__getitem__(key) if is_scalar(result): diff --git a/pandas/core/indexes/period.py b/pandas/core/indexes/period.py index 051c5ef3262ef..fd92f0cbcabb1 100644 --- a/pandas/core/indexes/period.py +++ b/pandas/core/indexes/period.py @@ -8,8 +8,7 @@ from pandas._libs.tslibs import NaT, iNaT, resolution from pandas._libs.tslibs.period import ( DIFFERENT_FREQ, IncompatibleFrequency, Period) -from pandas.util._decorators import ( - Appender, Substitution, cache_readonly, deprecate_kwarg) +from pandas.util._decorators import Appender, Substitution, cache_readonly from pandas.core.dtypes.common import ( is_bool_dtype, is_datetime64_any_dtype, is_float, is_float_dtype, @@ -472,34 +471,6 @@ def _int64index(self): # ------------------------------------------------------------------------ # Index Methods - @deprecate_kwarg(old_arg_name='n', new_arg_name='periods') - def shift(self, periods): - """ - Shift index by desired number of increments. - - This method is for shifting the values of period indexes - by a specified time increment. - - Parameters - ---------- - periods : int, default 1 - Number of periods (or increments) to shift by, - can be positive or negative. - - .. versionchanged:: 0.24.0 - - Returns - ------- - pandas.PeriodIndex - Shifted index. - - See Also - -------- - DatetimeIndex.shift : Shift values of DatetimeIndex. - """ - i8values = self._data._time_shift(periods) - return self._simple_new(i8values, name=self.name, freq=self.freq) - def _coerce_scalar_to_index(self, item): """ we need to coerce a scalar to a compat for our index type From b0fb9643dd19d2cb64c736cb6d7199bd97a41f68 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Mon, 31 Dec 2018 12:38:24 -0800 Subject: [PATCH 3/6] revert move of offset prop --- pandas/core/indexes/datetimelike.py | 24 ------------------------ pandas/core/indexes/datetimes.py | 22 ++++++++++++++++++++++ 2 files changed, 22 insertions(+), 24 deletions(-) diff --git a/pandas/core/indexes/datetimelike.py b/pandas/core/indexes/datetimelike.py index af4df37d50789..3b6e10de1f4ff 100644 --- a/pandas/core/indexes/datetimelike.py +++ b/pandas/core/indexes/datetimelike.py @@ -100,30 +100,6 @@ def wrapper(self, other): # ------------------------------------------------------------------------ - @property - def offset(self): - """ - get/set the frequency of the instance - """ - msg = ('{cls}.offset has been deprecated and will be removed ' - 'in a future version; use {cls}.freq instead.' - .format(cls=type(self).__name__)) - warnings.warn(msg, FutureWarning, stacklevel=2) - return self.freq - - @offset.setter - def offset(self, value): - """ - get/set the frequency of the instance - """ - msg = ('{cls}.offset has been deprecated and will be removed ' - 'in a future version; use {cls}.freq instead.' - .format(cls=type(self).__name__)) - warnings.warn(msg, FutureWarning, stacklevel=2) - self.freq = value - - # ------------------------------------------------------------------------ - def equals(self, other): """ Determines if two Index objects contain the same elements. diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py index 2be47b5b11461..d5c11ad364743 100644 --- a/pandas/core/indexes/datetimes.py +++ b/pandas/core/indexes/datetimes.py @@ -1180,6 +1180,28 @@ def tz(self, value): _has_same_tz = ea_passthrough("_has_same_tz") __array__ = ea_passthrough("__array__") + @property + def offset(self): + """ + get/set the frequency of the instance + """ + msg = ('{cls}.offset has been deprecated and will be removed ' + 'in a future version; use {cls}.freq instead.' + .format(cls=type(self).__name__)) + warnings.warn(msg, FutureWarning, stacklevel=2) + return self.freq + + @offset.setter + def offset(self, value): + """ + get/set the frequency of the instance + """ + msg = ('{cls}.offset has been deprecated and will be removed ' + 'in a future version; use {cls}.freq instead.' + .format(cls=type(self).__name__)) + warnings.warn(msg, FutureWarning, stacklevel=2) + self.freq = value + def __getitem__(self, key): result = self._eadata.__getitem__(key) if is_scalar(result): From 469d91c33ff91cf839ed7da88f4018f3a95df4a0 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Mon, 31 Dec 2018 13:22:47 -0800 Subject: [PATCH 4/6] Address comments --- pandas/core/arrays/datetimelike.py | 2 -- pandas/core/arrays/datetimes.py | 9 ++------- pandas/core/indexes/datetimelike.py | 13 +++++++++++++ pandas/core/indexes/datetimes.py | 29 +++-------------------------- pandas/core/indexes/period.py | 4 ---- 5 files changed, 18 insertions(+), 39 deletions(-) diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index a37fa614a65ba..98a1f1b925447 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -1138,8 +1138,6 @@ def _time_shift(self, periods, freq=None): freq = frequencies.to_offset(freq) offset = periods * freq result = self + offset - if getattr(self, 'tz', None): - result._dtype = self._dtype return result if periods == 0: diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index 0b38d34a2ab33..3f32b7b7dcea9 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -210,7 +210,7 @@ class DatetimeArrayMixin(dtl.DatetimeLikeArrayMixin, # Constructors _attributes = ["freq", "tz"] - _dtype = None + _dtype = None # type: Union[np.dtype, DatetimeTZDtype] _freq = None @classmethod @@ -406,11 +406,6 @@ def dtype(self): """ return self._dtype - @property - def _tz(self): - # TODO: Can we get rid of the private version of this? - return getattr(self._dtype, "tz", None) - @property def tz(self): """ @@ -422,7 +417,7 @@ def tz(self): Returns None when the array is tz-naive. """ # GH 18595 - return self._tz + return getattr(self._dtype, "tz", None) @tz.setter def tz(self, value): diff --git a/pandas/core/indexes/datetimelike.py b/pandas/core/indexes/datetimelike.py index 3b6e10de1f4ff..25cd5cda9989c 100644 --- a/pandas/core/indexes/datetimelike.py +++ b/pandas/core/indexes/datetimelike.py @@ -70,6 +70,15 @@ class DatetimeIndexOpsMixin(ExtensionOpsMixin): _maybe_mask_results = ea_passthrough("_maybe_mask_results") __iter__ = ea_passthrough("__iter__") + @property + def freq(self): + return self._eadata.freq + + @freq.setter + def freq(self, value): + # validation is handled by _eadata setter + self._eadata.freq = value + @property def freqstr(self): return self._eadata.freqstr @@ -98,6 +107,10 @@ def wrapper(self, other): wrapper.__name__ = '__{}__'.format(op.__name__) return wrapper + @property + def _ndarray_values(self): + return self._eadata._ndarray_values + # ------------------------------------------------------------------------ def equals(self, other): diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py index d5c11ad364743..70bc7fb2e295b 100644 --- a/pandas/core/indexes/datetimes.py +++ b/pandas/core/indexes/datetimes.py @@ -1132,38 +1132,15 @@ def slice_indexer(self, start=None, end=None, step=None, kind=None): def _data(self): return self._eadata._data - @property - def _freq(self): - return self._eadata._freq - - @_freq.setter - def _freq(self, value): - # Validation will be handled by _eadata setter - self._eadata._freq = value - - @property - def freq(self): - return self._freq - - @freq.setter - def freq(self, value): - # validation is handled by _eadata setter - self._eadata.freq = value - - @property - def _tz(self): - return self._eadata._tz - @property def tz(self): # GH#18595 - return self._tz + return self._eadata.tz @tz.setter def tz(self, value): - # GH 3746: Prevent localizing or converting the index by setting tz - raise AttributeError("Cannot directly set timezone. Use tz_localize() " - "or tz_convert() as appropriate") + # GH#3746; DatetimeArray will raise to disallow setting + self._eadata.tz = value tzinfo = tz diff --git a/pandas/core/indexes/period.py b/pandas/core/indexes/period.py index fd92f0cbcabb1..a915f24e3c87f 100644 --- a/pandas/core/indexes/period.py +++ b/pandas/core/indexes/period.py @@ -287,10 +287,6 @@ def _simple_new(cls, values, name=None, freq=None, **kwargs): def _eadata(self): return self._data - @property - def _ndarray_values(self): - return self._data._ndarray_values - @property def values(self): return np.asarray(self) From c8bd4518377460ef0c7967b36c4da3503726793a Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Mon, 31 Dec 2018 15:10:22 -0800 Subject: [PATCH 5/6] remove redundant props --- pandas/core/indexes/timedeltas.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/pandas/core/indexes/timedeltas.py b/pandas/core/indexes/timedeltas.py index 558a466297e2a..6206a6a615d64 100644 --- a/pandas/core/indexes/timedeltas.py +++ b/pandas/core/indexes/timedeltas.py @@ -283,23 +283,6 @@ def _format_native_types(self, na_rep='NaT', date_format=None, **kwargs): def _data(self): return self._eadata._data - @property - def _freq(self): - return self._eadata._freq - - @_freq.setter - def _freq(self, value): - self._eadata._freq = value - - @property - def freq(self): - return self._freq - - @freq.setter - def freq(self, value): - # Validation will be done in _eadata setter - self._eadata.freq = value - __mul__ = _make_wrapped_arith_op("__mul__") __rmul__ = _make_wrapped_arith_op("__rmul__") __floordiv__ = _make_wrapped_arith_op("__floordiv__") From 1456b6aadf897045266fd95ddb75a60ab60ddcc7 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Mon, 31 Dec 2018 15:39:43 -0800 Subject: [PATCH 6/6] set _eadata._dtype --- pandas/core/indexes/datetimes.py | 24 +++++++-------------- pandas/tests/arithmetic/test_timedelta64.py | 2 +- 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py index 70bc7fb2e295b..5ed8bd45a6aff 100644 --- a/pandas/core/indexes/datetimes.py +++ b/pandas/core/indexes/datetimes.py @@ -491,14 +491,9 @@ def union(self, other): else: result = Index.union(this, other) if isinstance(result, DatetimeIndex): - if this.tz is not None: - # TODO: can this be simplified? It used to just be - # `result._tz = timezones.tz_standardize(this.tz)` - tz = timezones.tz_standardize(this.tz) - if result.tz is None: - result = result.tz_localize(tz) - else: - result = result.tz_convert(tz) + # TODO: we shouldn't be setting attributes like this; + # in all the tests this equality already holds + result._eadata._dtype = this.dtype if (result.freq is None and (this.freq is not None or other.freq is not None)): result.freq = to_offset(result.inferred_freq) @@ -526,15 +521,12 @@ def union_many(self, others): if this._can_fast_union(other): this = this._fast_union(other) else: - tz = this.tz + dtype = this.dtype this = Index.union(this, other) - if isinstance(this, DatetimeIndex) and tz is not None: - # TODO: can this be simplified? it used to just be - # `this._tz = tz_standardize(tz)` - if this.tz is None: - this = this.tz_localize(timezones.tz_standardize(tz)) - else: - this = this.tz_convert(timezones.tz_standardize(tz)) + if isinstance(this, DatetimeIndex): + # TODO: we shouldn't be setting attributes like this; + # in all the tests this equality already holds + this._eadata._dtype = dtype return this def _can_fast_union(self, other): diff --git a/pandas/tests/arithmetic/test_timedelta64.py b/pandas/tests/arithmetic/test_timedelta64.py index f672baed944fc..dea4940eb3180 100644 --- a/pandas/tests/arithmetic/test_timedelta64.py +++ b/pandas/tests/arithmetic/test_timedelta64.py @@ -1476,7 +1476,7 @@ def test_tdi_rmul_arraylike(self, other, box_with_array): tdi = TimedeltaIndex(['1 Day'] * 10) expected = timedelta_range('1 days', '10 days') - expected._freq = None + expected._eadata._freq = None tdi = tm.box_expected(tdi, box) expected = tm.box_expected(expected, xbox)