Skip to content

Closes #36541 (BUG: ValueError: cannot convert float NaN to integer when resetting MultiIndex with NaT values) #36563

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 19 commits into from
Oct 7, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions doc/source/whatsnew/v1.2.0.rst
Original file line number Diff line number Diff line change
Expand Up @@ -370,6 +370,7 @@ MultiIndex
^^^^^^^^^^

- Bug in :meth:`DataFrame.xs` when used with :class:`IndexSlice` raises ``TypeError`` with message ``"Expected label or tuple of labels"`` (:issue:`35301`)
- Bug in :meth:`DataFrame.reset_index` with ``NaT`` values in index raises ``ValueError`` with message ``"cannot convert float NaN to integer"`` (:issue:`36541`)
-

I/O
Expand Down
7 changes: 6 additions & 1 deletion pandas/core/dtypes/cast.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@
ABCSeries,
)
from pandas.core.dtypes.inference import is_list_like
from pandas.core.dtypes.missing import isna, notna
from pandas.core.dtypes.missing import is_valid_nat_for_dtype, isna, notna

if TYPE_CHECKING:
from pandas import Series
Expand Down Expand Up @@ -1559,6 +1559,11 @@ def construct_1d_arraylike_from_scalar(
dtype = np.dtype("object")
if not isna(value):
value = ensure_str(value)
elif dtype.kind in ["M", "m"] and is_valid_nat_for_dtype(value, dtype):
# GH36541: can't fill array directly with pd.NaT
# > np.empty(10, dtype="datetime64[64]").fill(pd.NaT)
# ValueError: cannot convert float NaN to integer
value = np.datetime64("NaT")

subarr = np.empty(length, dtype=dtype)
subarr.fill(value)
Expand Down
29 changes: 29 additions & 0 deletions pandas/tests/series/indexing/test_multiindex.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
""" test get/set & misc """

import pytest

import pandas as pd
from pandas import MultiIndex, Series
import pandas._testing as tm


def test_access_none_value_in_multiindex():
Expand All @@ -20,3 +22,30 @@ def test_access_none_value_in_multiindex():
s = Series([1] * len(midx), dtype=object, index=midx)
result = s.loc[("Level1", "Level2_a")]
assert result == 1


@pytest.mark.parametrize(
"ix_data, exp_data",
[
(
[(pd.NaT, 1), (pd.NaT, 2)],
{"a": [pd.NaT, pd.NaT], "b": [1, 2], "x": [11, 12]},
),
(
[(pd.NaT, 1), (pd.Timestamp("2020-01-01"), 2)],
{"a": [pd.NaT, pd.Timestamp("2020-01-01")], "b": [1, 2], "x": [11, 12]},
),
(
[(pd.NaT, 1), (pd.Timedelta(123, "d"), 2)],
{"a": [pd.NaT, pd.Timedelta(123, "d")], "b": [1, 2], "x": [11, 12]},
),
],
)
def test_nat_multi_index(ix_data, exp_data):
# GH36541: that reset_index() does not raise ValueError
ix = pd.MultiIndex.from_tuples(ix_data, names=["a", "b"])
result = pd.DataFrame({"x": [11, 12]}, index=ix)
result = result.reset_index()

expected = pd.DataFrame(exp_data)
tm.assert_frame_equal(result, expected)