-
-
Notifications
You must be signed in to change notification settings - Fork 18.4k
BUG: various groupby ewm times issues #40952
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 7 commits
e7a9d31
8e3ed3b
b3e7e99
b7f38e3
e1ecb58
c5e80d1
91f7390
f2dc759
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -268,15 +268,7 @@ def __init__( | |
) | ||
if isna(self.times).any(): | ||
raise ValueError("Cannot convert NaT values to integer") | ||
# error: Item "str" of "Union[str, ndarray, FrameOrSeries, None]" has no | ||
# attribute "view" | ||
# error: Item "None" of "Union[str, ndarray, FrameOrSeries, None]" has no | ||
# attribute "view" | ||
_times = np.asarray( | ||
self.times.view(np.int64), dtype=np.float64 # type: ignore[union-attr] | ||
) | ||
_halflife = float(Timedelta(self.halflife).value) | ||
self._deltas = np.diff(_times) / _halflife | ||
self._deltas = self._calculate_deltas(self.times, self.halflife) | ||
# Halflife is no longer applicable when calculating COM | ||
# But allow COM to still be calculated if the user passes other decay args | ||
if common.count_not_none(self.com, self.span, self.alpha) > 0: | ||
|
@@ -303,6 +295,38 @@ def __init__( | |
self.alpha, | ||
) | ||
|
||
@staticmethod | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. just make this a module level function |
||
def _calculate_deltas( | ||
times: str | np.ndarray | FrameOrSeries | None, | ||
halflife: float | TimedeltaConvertibleTypes | None, | ||
) -> np.ndarray: | ||
""" | ||
Return the diff of the times divided by the half-life. These values are used in | ||
the calculation of the ewm mean. | ||
|
||
Parameters | ||
---------- | ||
times : str, np.ndarray, Series, default None | ||
Times corresponding to the observations. Must be monotonically increasing | ||
and ``datetime64[ns]`` dtype. | ||
halflife : float, str, timedelta, optional | ||
Half-life specifying the decay | ||
|
||
Returns | ||
------- | ||
np.ndarray | ||
Diff of the times divided by the half-life | ||
""" | ||
# error: Item "str" of "Union[str, ndarray, FrameOrSeries, None]" has no | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. can you give a doc-string, describe the params and what this is doing |
||
# attribute "view" | ||
# error: Item "None" of "Union[str, ndarray, FrameOrSeries, None]" has no | ||
# attribute "view" | ||
_times = np.asarray( | ||
times.view(np.int64), dtype=np.float64 # type: ignore[union-attr] | ||
) | ||
_halflife = float(Timedelta(halflife).value) | ||
return np.diff(_times) / _halflife | ||
|
||
def _get_window_indexer(self) -> BaseIndexer: | ||
""" | ||
Return an indexer class that will compute the window start and end bounds | ||
|
@@ -585,6 +609,17 @@ class ExponentialMovingWindowGroupby(BaseWindowGroupby, ExponentialMovingWindow) | |
|
||
_attributes = ExponentialMovingWindow._attributes + BaseWindowGroupby._attributes | ||
|
||
def __init__(self, obj, *args, _grouper=None, **kwargs): | ||
super().__init__(obj, *args, _grouper=_grouper, **kwargs) | ||
|
||
if not obj.empty and self.times is not None: | ||
# sort the times and recalculate the deltas according to the groups | ||
groupby_order = np.concatenate(list(self._grouper.indices.values())) | ||
self._deltas = self._calculate_deltas( | ||
self.times.take(groupby_order), # type: ignore[union-attr] | ||
self.halflife, | ||
) | ||
|
||
def _get_window_indexer(self) -> GroupbyIndexer: | ||
""" | ||
Return an indexer class that will compute the window start and end bounds | ||
|
@@ -628,10 +663,7 @@ def mean(self, engine=None, engine_kwargs=None): | |
""" | ||
if maybe_use_numba(engine): | ||
groupby_ewma_func = generate_numba_groupby_ewma_func( | ||
engine_kwargs, | ||
self._com, | ||
self.adjust, | ||
self.ignore_na, | ||
engine_kwargs, self._com, self.adjust, self.ignore_na, self._deltas | ||
) | ||
return self._apply( | ||
groupby_ewma_func, | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -85,6 +85,7 @@ def generate_numba_groupby_ewma_func( | |
com: float, | ||
adjust: bool, | ||
ignore_na: bool, | ||
deltas: np.ndarray, | ||
): | ||
""" | ||
Generate a numba jitted groupby ewma function specified by values | ||
|
@@ -97,6 +98,7 @@ def generate_numba_groupby_ewma_func( | |
com : float | ||
adjust : bool | ||
ignore_na : bool | ||
deltas : numpy.ndarray | ||
|
||
Returns | ||
------- | ||
|
@@ -141,7 +143,7 @@ def groupby_ewma( | |
|
||
if is_observation or not ignore_na: | ||
|
||
old_wt *= old_wt_factor | ||
old_wt *= old_wt_factor ** deltas[start + j - 1] | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. why -1? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We want to use I could introduce a new variable There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. no its fine, if you'd just add some documentation to this effect for future readers |
||
if is_observation: | ||
|
||
# avoid numerical errors on constant series | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
why is this not
s:e
? (like sub_vals)There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Deltas are the (scaled) np.diff of the times vector. so
len(deltas) = len(times) - 1 = len(vals) - 1
.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
kk can you add this comment on the shape of sub_vals vs sub_deltas for future readers