Skip to content

CLN: Make Series._values match Index._values #31182

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 16 commits into from
Jan 28, 2020
Merged
Show file tree
Hide file tree
Changes from 7 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion pandas/core/apply.py
Original file line number Diff line number Diff line change
Expand Up @@ -278,7 +278,9 @@ def apply_standard(self):
if (
self.result_type in ["reduce", None]
and not self.dtypes.apply(is_extension_array_dtype).any()
# Disallow complex_internals since libreduction shortcut
# Disallow dtypes that have blocks backed by EAs
and not self.dtypes.apply(lambda x: x.kind in ["m", "M"]).any()
# Disallow MultiIndex since libreduction shortcut
# cannot handle MultiIndex
and not isinstance(self.agg_axis, ABCMultiIndex)
):
Expand Down
4 changes: 4 additions & 0 deletions pandas/core/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -1265,6 +1265,10 @@ def unique(self):
if hasattr(values, "unique"):

result = values.unique()
if self.dtype.kind in ["m", "M"] and isinstance(self, ABCSeries):
# GH#31182 Series._values returns EA, unpack for backward-compat
if getattr(self.dtype, "tz", None) is None:
result = np.asarray(result)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you add a comment here on why this is needed

else:
result = unique1d(values)

Expand Down
22 changes: 16 additions & 6 deletions pandas/core/internals/blocks.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import pandas._libs.internals as libinternals
from pandas._libs.tslibs import Timedelta, conversion
from pandas._libs.tslibs.timezones import tz_compare
from pandas.util._decorators import cache_readonly
from pandas.util._validators import validate_bool_kwarg

from pandas.core.dtypes.cast import (
Expand Down Expand Up @@ -220,6 +221,10 @@ def array_values(self) -> ExtensionArray:
"""
The array that Series.array returns. Always an ExtensionArray.
"""
return self._ea_values

@cache_readonly
def _ea_values(self) -> ExtensionArray:
return PandasArray(self.values)

def get_values(self, dtype=None):
Expand Down Expand Up @@ -2138,6 +2143,16 @@ def get_values(self, dtype=None):
return result.reshape(self.values.shape)
return self.values

def array_values(self):
return self._ea_values

def internal_values(self):
return self._ea_values

@cache_readonly
def _ea_values(self):
return self._holder._simple_new(self.values)


class DatetimeBlock(DatetimeLikeBlockMixin, Block):
__slots__ = ()
Expand Down Expand Up @@ -2260,9 +2275,6 @@ def set(self, locs, values):
def external_values(self):
return np.asarray(self.values.astype("datetime64[ns]", copy=False))

def array_values(self) -> ExtensionArray:
return DatetimeArray._simple_new(self.values)


class DatetimeTZBlock(ExtensionBlock, DatetimeBlock):
""" implement a datetime64 block with a tz attribute """
Expand All @@ -2271,6 +2283,7 @@ class DatetimeTZBlock(ExtensionBlock, DatetimeBlock):
is_datetimetz = True
is_extension = True

internal_values = Block.internal_values
_can_hold_element = DatetimeBlock._can_hold_element
to_native_types = DatetimeBlock.to_native_types
fill_value = np.datetime64("NaT", "ns")
Expand Down Expand Up @@ -2520,9 +2533,6 @@ def to_native_types(self, slicer=None, na_rep=None, quoting=None, **kwargs):
def external_values(self):
return np.asarray(self.values.astype("timedelta64[ns]", copy=False))

def array_values(self) -> ExtensionArray:
return TimedeltaArray._simple_new(self.values)


class BoolBlock(NumericBlock):
__slots__ = ()
Expand Down
3 changes: 2 additions & 1 deletion pandas/core/series.py
Original file line number Diff line number Diff line change
Expand Up @@ -515,8 +515,9 @@ def _values(self):
----------- | ------------- | ------------- | ------------- | --------------- |
Numeric | ndarray | ndarray | PandasArray | ndarray |
Category | Categorical | Categorical | Categorical | ndarray[int] |
dt64[ns] | ndarray[M8ns] | ndarray[M8ns] | DatetimeArray | ndarray[M8ns] |
dt64[ns] | ndarray[M8ns] | DatetimeArray | DatetimeArray | ndarray[M8ns] |
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A bit above (beginning of the docstring), the sentence "This are the values as stored in the Block" is no longer adequate I think?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The sentence I am quoting (the second line of the docstring) still needs to be updated

dt64[ns tz] | ndarray[M8ns] | DatetimeArray | DatetimeArray | ndarray[M8ns] |
td64[ns] | ndarray[m8ns] | TimedeltaArray| ndarray[m8bs] | ndarray[m8ns] |
Period | ndarray[obj] | PeriodArray | PeriodArray | ndarray[int] |
Nullable | EA | EA | EA | ndarray |

Expand Down
2 changes: 1 addition & 1 deletion pandas/tests/indexes/datetimes/test_tools.py
Original file line number Diff line number Diff line change
Expand Up @@ -1583,7 +1583,7 @@ def test_string_na_nat_conversion(self, cache):
for i in range(5):
x = series[i]
if isna(x):
expected[i] = iNaT
expected[i] = pd.NaT
else:
expected[i] = to_datetime(x, cache=cache)

Expand Down
15 changes: 13 additions & 2 deletions pandas/tests/reductions/test_reductions.py
Original file line number Diff line number Diff line change
Expand Up @@ -316,7 +316,12 @@ def test_invalid_td64_reductions(self, opname):
)
td = s.diff()

msg = "reduction operation '{op}' not allowed for this dtype"
msg = "|".join(
[
"reduction operation '{op}' not allowed for this dtype",
r"cannot perform {op} with type timedelta64\[ns\]",
]
)
msg = msg.format(op=opname)

with pytest.raises(TypeError, match=msg):
Expand Down Expand Up @@ -648,7 +653,13 @@ def test_ops_consistency_on_empty(self, method):
# timedelta64[ns]
tdser = Series([], dtype="m8[ns]")
if method == "var":
with pytest.raises(TypeError, match="operation 'var' not allowed"):
msg = "|".join(
[
"operation 'var' not allowed",
r"cannot perform var with type timedelta64\[ns\]",
]
)
with pytest.raises(TypeError, match=msg):
getattr(tdser, method)()
else:
result = getattr(tdser, method)()
Expand Down