Skip to content

BUG: transpose inferring dtype for dt in object column #51565

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
Mar 7, 2023
Merged
Show file tree
Hide file tree
Changes from 6 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion doc/source/whatsnew/v2.1.0.rst
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,7 @@ Groupby/resample/rolling

Reshaping
^^^^^^^^^
-
- Bug in :meth:`DataFrame.transpose` inferring dtype for object column (:issue:`51546`)
-

Sparse
Expand Down
1 change: 1 addition & 0 deletions pandas/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -1919,6 +1919,7 @@ def using_copy_on_write() -> bool:
"""
Fixture to check if Copy-on-Write is enabled.
"""
pd.options.mode.copy_on_write = True
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is this intentional?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nope thx, already removed. Snuck in

return pd.options.mode.copy_on_write and pd.options.mode.data_manager == "block"


Expand Down
10 changes: 8 additions & 2 deletions pandas/core/frame.py
Original file line number Diff line number Diff line change
Expand Up @@ -3555,7 +3555,11 @@ def transpose(self, *args, copy: bool = False) -> DataFrame:
new_vals = new_vals.copy()

result = self._constructor(
new_vals, index=self.columns, columns=self.index, copy=False
new_vals,
index=self.columns,
columns=self.index,
copy=False,
dtype=new_vals.dtype,
)
if using_copy_on_write() and len(self) > 0:
result._mgr.add_references(self._mgr) # type: ignore[arg-type]
Expand All @@ -3577,7 +3581,9 @@ def transpose(self, *args, copy: bool = False) -> DataFrame:
new_arr = self.values.T
if copy:
new_arr = new_arr.copy()
result = self._constructor(new_arr, index=self.columns, columns=self.index)
result = self._constructor(
new_arr, index=self.columns, columns=self.index, dtype=new_arr.dtype
)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

do we need to pass copy=False here (possibly just to be future-safe)?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yep, I’ll add it in the copy=True pr.

actually, it probably makes sense to go through all usages of constructor and pass copy=False for arrays. Will take a look how many there are


return result.__finalize__(self, method="transpose")

Expand Down
45 changes: 43 additions & 2 deletions pandas/tests/frame/methods/test_transpose.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
DataFrame,
DatetimeIndex,
IntervalIndex,
Series,
Timestamp,
date_range,
timedelta_range,
)
Expand Down Expand Up @@ -63,7 +65,7 @@ def test_transpose_tzaware_2col_mixed_tz(self):
df4 = DataFrame({"A": dti, "B": dti2})
assert (df4.dtypes == [dti.dtype, dti2.dtype]).all()
assert (df4.T.dtypes == object).all()
tm.assert_frame_equal(df4.T.T, df4)
tm.assert_frame_equal(df4.T.T, df4.astype(object))

@pytest.mark.parametrize("tz", [None, "America/New_York"])
def test_transpose_preserves_dtindex_equality_with_dst(self, tz):
Expand All @@ -83,7 +85,7 @@ def test_transpose_object_to_tzaware_mixed_tz(self):
df2 = DataFrame([dti, dti2])
assert (df2.dtypes == object).all()
res2 = df2.T
assert (res2.dtypes == [dti.dtype, dti2.dtype]).all()
assert (res2.dtypes == object).all()

def test_transpose_uint64(self, uint64_frame):
result = uint64_frame.T
Expand Down Expand Up @@ -128,3 +130,42 @@ def test_transpose_get_view_dt64tzget_view(self):

rtrip = result._mgr.blocks[0].values
assert np.shares_memory(arr._ndarray, rtrip._ndarray)

def test_transpose_not_inferring_dt(self):
# GH#51546
df = DataFrame(
{
"a": [Timestamp("2019-12-31"), Timestamp("2019-12-31")],
},
dtype=object,
)
result = df.T
expected = DataFrame(
[[Timestamp("2019-12-31"), Timestamp("2019-12-31")]],
columns=[0, 1],
index=["a"],
dtype=object,
)
tm.assert_frame_equal(result, expected)

def test_transpose_not_inferring_dt_mixed_blocks(self):
# GH#51546
df = DataFrame(
{
"a": Series(
[Timestamp("2019-12-31"), Timestamp("2019-12-31")], dtype=object
),
"b": [Timestamp("2019-12-31"), Timestamp("2019-12-31")],
}
)
result = df.T
expected = DataFrame(
[
[Timestamp("2019-12-31"), Timestamp("2019-12-31")],
[Timestamp("2019-12-31"), Timestamp("2019-12-31")],
],
columns=[0, 1],
index=["a", "b"],
dtype=object,
)
tm.assert_frame_equal(result, expected)
2 changes: 1 addition & 1 deletion pandas/tests/groupby/test_quantile.py
Original file line number Diff line number Diff line change
Expand Up @@ -464,7 +464,7 @@ def test_groupby_quantile_dt64tz_period():

# Check that we match the group-by-group result
exp = {i: df.iloc[i::5].quantile(0.5) for i in range(5)}
expected = DataFrame(exp).T
expected = DataFrame(exp).T.infer_objects()
expected.index = expected.index.astype(np.int_)

tm.assert_frame_equal(result, expected)