Skip to content

PERF: slow tests #44727

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 9 commits into from
Dec 3, 2021
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 12 additions & 1 deletion pandas/core/arrays/datetimelike.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
Timestamp,
delta_to_nanoseconds,
iNaT,
ints_to_pydatetime,
to_offset,
)
from pandas._libs.tslibs.fields import (
Expand Down Expand Up @@ -332,8 +333,9 @@ def __getitem__(
only handle list-likes, slices, and integer scalars
"""
# Use cast as we know we will get back a DatetimeLikeArray or DTScalar
# GH#44624 skip evaluating the Union at runtime for performance.
result = cast(
Union[DatetimeLikeArrayT, DTScalarOrNaT], super().__getitem__(key)
"Union[DatetimeLikeArrayT, DTScalarOrNaT]", super().__getitem__(key)
)
if lib.is_scalar(result):
return result
Expand Down Expand Up @@ -403,6 +405,15 @@ def astype(self, dtype, copy: bool = True):
dtype = pandas_dtype(dtype)

if is_object_dtype(dtype):
if self.dtype.kind == "M":
# *much* faster than self._box_values
# for e.g. test_get_loc_tuple_monotonic_above_size_cutoff
i8data = self.asi8.ravel()
converted = ints_to_pydatetime(
i8data, tz=self.tz, freq=self.freq, box="timestamp"
)
return converted.reshape(self.shape)

return self._box_values(self.asi8.ravel()).reshape(self.shape)

elif isinstance(dtype, ExtensionDtype):
Expand Down
17 changes: 12 additions & 5 deletions pandas/core/dtypes/missing.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@
is_integer_dtype,
is_object_dtype,
is_scalar,
is_string_dtype,
needs_i8_conversion,
)
from pandas.core.dtypes.dtypes import (
Expand Down Expand Up @@ -242,7 +241,7 @@ def _isna_array(values: ArrayLike, inf_as_na: bool = False):
result = libmissing.isnaobj(values.to_numpy(), inf_as_na=inf_as_na)
else:
result = values.isna()
elif is_string_dtype(dtype):
elif _is_string_dtype(dtype):
result = _isna_string_dtype(values, inf_as_na=inf_as_na)
elif needs_i8_conversion(dtype):
# this is the NaT pattern
Expand Down Expand Up @@ -376,7 +375,10 @@ def isna_compat(arr, fill_value=np.nan) -> bool:


def array_equivalent(
left, right, strict_nan: bool = False, dtype_equal: bool = False
left: np.ndarray,
right: np.ndarray,
strict_nan: bool = False,
dtype_equal: bool = False,
) -> bool:
"""
True if two arrays, left and right, have equal non-NaN elements, and NaNs
Expand Down Expand Up @@ -421,13 +423,13 @@ def array_equivalent(

if dtype_equal:
# fastpath when we require that the dtypes match (Block.equals)
if is_float_dtype(left.dtype) or is_complex_dtype(left.dtype):
if left.dtype.kind in ["f", "c"]:
return _array_equivalent_float(left, right)
elif is_datetimelike_v_numeric(left.dtype, right.dtype):
return False
elif needs_i8_conversion(left.dtype):
return _array_equivalent_datetimelike(left, right)
elif is_string_dtype(left.dtype):
elif _is_string_dtype(left.dtype):
# TODO: fastpath for pandas' StringDtype
return _array_equivalent_object(left, right, strict_nan)
else:
Expand Down Expand Up @@ -644,3 +646,8 @@ def is_valid_na_for_dtype(obj, dtype: DtypeObj) -> bool:

# fallback, default to allowing NaN, None, NA, NaT
return not isinstance(obj, (np.datetime64, np.timedelta64, Decimal))


def _is_string_dtype(dtype: np.dtype) -> bool:
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

hmm can you simply move this to dtypes.core.common next to the other one

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure. most recent commit also added an improvement for random_text

# Faster implementation of dtypes.common.is_string_dtype
return dtype == object or dtype.kind in "SU"