Skip to content

Fix Regression when using sum/cumsum on Groupby objects #44526

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 26 additions & 8 deletions pandas/_libs/groupby.pyx
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,10 @@ from numpy cimport (
uint32_t,
uint64_t,
)
from numpy.math cimport NAN
from numpy.math cimport (
NAN,
isinf,
)

cnp.import_array()

Expand All @@ -51,7 +54,14 @@ from pandas._libs.missing cimport checknull
cdef int64_t NPY_NAT = get_nat()
_int64_max = np.iinfo(np.int64).max

cdef float64_t NaN = <float64_t>np.NaN
cdef:
float32_t MINfloat32 = np.NINF
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we have some comment definitions of these elsewhere in a pxd

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

FYI these are being added independntly here: #44522

float64_t MINfloat64 = np.NINF

float32_t MAXfloat32 = np.inf
float64_t MAXfloat64 = np.inf

float64_t NaN = <float64_t>np.NaN

cdef enum InterpolationEnumType:
INTERPOLATION_LINEAR,
Expand Down Expand Up @@ -251,13 +261,18 @@ def group_cumsum(numeric_t[:, ::1] out,

# For floats, use Kahan summation to reduce floating-point
# error (https://en.wikipedia.org/wiki/Kahan_summation_algorithm)
if numeric_t == float32_t or numeric_t == float64_t:
if numeric_t is float32_t or numeric_t is float64_t:
if val == val:
y = val - compensation[lab, j]
t = accum[lab, j] + y
compensation[lab, j] = t - accum[lab, j] - y
accum[lab, j] = t
out[i, j] = t
# if val or accum are inf/-inf don't use kahan
if isinf(val) or isinf(accum[lab, j]):
accum[lab, j] += val
out[i, j] = accum[lab, j]
else:
y = val - compensation[lab, j]
t = accum[lab, j] + y
compensation[lab, j] = t - accum[lab, j] - y
accum[lab, j] = t
out[i, j] = t
else:
out[i, j] = NaN
if not skipna:
Expand Down Expand Up @@ -557,6 +572,9 @@ def group_add(add_t[:, ::1] out,
for j in range(K):
val = values[i, j]

if (val == MAXfloat64) or (val == MINfloat64):
sumx[lab, j] = val
break
# not nan
# With dt64/td64 values, values have been cast to float64
# instead if int64 for group_add, but the logic
Expand Down
8 changes: 8 additions & 0 deletions pandas/_libs/window/aggregations.pyx
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,10 @@ cdef inline void add_sum(float64_t val, int64_t *nobs, float64_t *sum_x,
t = sum_x[0] + y
compensation[0] = t - sum_x[0] - y
sum_x[0] = t
if (val == MINfloat64) or (val == MAXfloat64):
sum_x[0] = val
nobs[0] = nobs[0] + 1
compensation[0] = 0


cdef inline void remove_sum(float64_t val, int64_t *nobs, float64_t *sum_x,
Expand All @@ -116,6 +120,10 @@ cdef inline void remove_sum(float64_t val, int64_t *nobs, float64_t *sum_x,
t = sum_x[0] + y
compensation[0] = t - sum_x[0] - y
sum_x[0] = t
if (val == MINfloat64) or (val == MAXfloat64):
sum_x[0] = val
nobs[0] = nobs[0] - 1
compensation[0] = 0


def roll_sum(const float64_t[:] values, ndarray[int64_t] start,
Expand Down
28 changes: 28 additions & 0 deletions pandas/tests/groupby/test_function.py
Original file line number Diff line number Diff line change
Expand Up @@ -1186,3 +1186,31 @@ def test_groupby_sum_timedelta_with_nat():
res = gb["b"].sum(min_count=2)
expected = Series([td3, pd.NaT], dtype="m8[ns]", name="b", index=expected.index)
tm.assert_series_equal(res, expected)


def test_sum_with_nan_inf():
df = DataFrame(
{"a": ["hello", "hello", "world", "world"], "b": [np.inf, 10, np.nan, 10]}
)
gb = df.groupby("a")
result = gb.sum()
expected = DataFrame(
[np.inf, 10], index=Index(["hello", "world"], name="a"), columns=["b"]
)
tm.assert_frame_equal(result, expected)


def test_cumsum_inf():
ser = Series([np.inf, 1, 1])

result = ser.groupby([1, 1, 1]).cumsum()
expected = Series([np.inf, np.inf, np.inf])
tm.assert_series_equal(result, expected)


def test_cumsum_ninf_inf():
ser = Series([np.inf, 1, 1, -np.inf, 1])

result = ser.groupby([1, 1, 1, 1, 1]).cumsum()
expected = Series([np.inf, np.inf, np.inf, np.nan, np.nan])
tm.assert_series_equal(result, expected)