From 4c02b909c270e906954716c900219df83621207e Mon Sep 17 00:00:00 2001 From: Brock Date: Sat, 2 Jan 2021 11:37:05 -0800 Subject: [PATCH 1/2] BUG: casting on concat with empties --- pandas/core/internals/concat.py | 6 ++++++ pandas/tests/indexing/test_partial.py | 3 ++- pandas/tests/reshape/concat/test_append.py | 6 +----- pandas/tests/reshape/concat/test_empty.py | 1 - 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/pandas/core/internals/concat.py b/pandas/core/internals/concat.py index 013e52248f5c4..a45933714ee96 100644 --- a/pandas/core/internals/concat.py +++ b/pandas/core/internals/concat.py @@ -318,6 +318,12 @@ def _concatenate_join_units( # Concatenating join units along ax0 is handled in _merge_blocks. raise AssertionError("Concatenating join units along axis0") + nonempties = [ + x for x in join_units if x.block is None or x.block.shape[concat_axis] > 0 + ] + if nonempties: + join_units = nonempties + empty_dtype, upcasted_na = _get_empty_dtype_and_na(join_units) to_concat = [ diff --git a/pandas/tests/indexing/test_partial.py b/pandas/tests/indexing/test_partial.py index d8dd08ea13341..f2d628c70ae62 100644 --- a/pandas/tests/indexing/test_partial.py +++ b/pandas/tests/indexing/test_partial.py @@ -154,7 +154,8 @@ def test_partial_setting_mixed_dtype(self): # columns will align df = DataFrame(columns=["A", "B"]) df.loc[0] = Series(1, index=range(4)) - tm.assert_frame_equal(df, DataFrame(columns=["A", "B"], index=[0])) + expected = DataFrame(columns=["A", "B"], index=[0], dtype=int) + tm.assert_frame_equal(df, expected) # columns will align df = DataFrame(columns=["A", "B"]) diff --git a/pandas/tests/reshape/concat/test_append.py b/pandas/tests/reshape/concat/test_append.py index ffeda703cd890..1a895aee98f0a 100644 --- a/pandas/tests/reshape/concat/test_append.py +++ b/pandas/tests/reshape/concat/test_append.py @@ -82,6 +82,7 @@ def test_append_length0_frame(self, sort): df5 = df.append(df3, sort=sort) expected = DataFrame(index=[0, 1], columns=["A", "B", "C"]) + expected["C"] = expected["C"].astype(np.float64) tm.assert_frame_equal(df5, expected) def test_append_records(self): @@ -340,16 +341,11 @@ def test_append_empty_frame_to_series_with_dateutil_tz(self): expected = DataFrame( [[np.nan, np.nan, 1.0, 2.0, date]], columns=["c", "d", "a", "b", "date"] ) - # These columns get cast to object after append - expected["c"] = expected["c"].astype(object) - expected["d"] = expected["d"].astype(object) tm.assert_frame_equal(result_a, expected) expected = DataFrame( [[np.nan, np.nan, 1.0, 2.0, date]] * 2, columns=["c", "d", "a", "b", "date"] ) - expected["c"] = expected["c"].astype(object) - expected["d"] = expected["d"].astype(object) result_b = result_a.append(s, ignore_index=True) tm.assert_frame_equal(result_b, expected) diff --git a/pandas/tests/reshape/concat/test_empty.py b/pandas/tests/reshape/concat/test_empty.py index dea04e98088e8..075785120677a 100644 --- a/pandas/tests/reshape/concat/test_empty.py +++ b/pandas/tests/reshape/concat/test_empty.py @@ -210,7 +210,6 @@ def test_concat_empty_df_object_dtype(self, dtype): df_2 = DataFrame(columns=df_1.columns) result = pd.concat([df_1, df_2], axis=0) expected = df_1.copy() - expected["EmptyCol"] = expected["EmptyCol"].astype(object) # TODO: why? tm.assert_frame_equal(result, expected) def test_concat_empty_dataframe_dtypes(self): From ccd6cf805f6a37bffceea56d46920ca8b319cd86 Mon Sep 17 00:00:00 2001 From: Brock Date: Sat, 2 Jan 2021 11:38:39 -0800 Subject: [PATCH 2/2] whatsnew --- doc/source/whatsnew/v1.3.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v1.3.0.rst b/doc/source/whatsnew/v1.3.0.rst index 5e84947cd42f1..b4a31635687e1 100644 --- a/doc/source/whatsnew/v1.3.0.rst +++ b/doc/source/whatsnew/v1.3.0.rst @@ -294,7 +294,7 @@ Groupby/resample/rolling Reshaping ^^^^^^^^^ - Bug in :meth:`DataFrame.unstack` with missing levels led to incorrect index names (:issue:`37510`) -- Bug in :func:`concat` incorrectly casting to ``object`` dtype in some cases when one or more of the operands is empty (:issue:`38843`) +- Bug in :func:`concat` incorrectly casting to ``object`` dtype in some cases when one or more of the operands is empty (:issue:`38843`, :issue:`38907`) -