Skip to content

Commit f7bda17

Browse files
authored
BUG: pd.concat dataframes with different datetime64 resolutions (#53641)
* BUG: pd.concat dataframes with different datetime64 resolutions * resolve mypy * retrigger checks * parametrize test * apply suggestion of jbrockmendel * add parametrization for test_concat_tz_series* * apply suggested changes by jbrockmendel
1 parent cb0a11e commit f7bda17

File tree

2 files changed

+51
-18
lines changed

2 files changed

+51
-18
lines changed

pandas/core/dtypes/concat.py

+10-1
Original file line numberDiff line numberDiff line change
@@ -129,7 +129,16 @@ def concat_compat(
129129
# i.e. isinstance(to_concat[0], ExtensionArray)
130130
to_concat_eas = cast("Sequence[ExtensionArray]", to_concat)
131131
cls = type(to_concat[0])
132-
return cls._concat_same_type(to_concat_eas)
132+
# GH#53640: eg. for datetime array, axis=1 but 0 is default
133+
# However, class method `_concat_same_type()` for some classes
134+
# may not support the `axis` keyword
135+
if ea_compat_axis or axis == 0:
136+
return cls._concat_same_type(to_concat_eas)
137+
else:
138+
return cls._concat_same_type(
139+
to_concat_eas,
140+
axis=axis, # type: ignore[call-arg]
141+
)
133142
else:
134143
to_concat_arrs = cast("Sequence[np.ndarray]", to_concat)
135144
result = np.concatenate(to_concat_arrs, axis=axis)

pandas/tests/reshape/concat/test_datetimes.py

+41-17
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,22 @@
1919
)
2020
import pandas._testing as tm
2121

22+
UNITS = ["s", "ms", "us", "ns"]
23+
24+
25+
@pytest.fixture(params=UNITS)
26+
def unit(request):
27+
return request.param
28+
29+
30+
unit2 = unit
31+
32+
33+
def _get_finer_unit(unit, unit2):
34+
if UNITS.index(unit) >= UNITS.index(unit2):
35+
return unit
36+
return unit2
37+
2238

2339
class TestDatetimeConcat:
2440
def test_concat_datetime64_block(self):
@@ -307,50 +323,58 @@ def test_concat_tz_series2(self):
307323
result = concat([x, y], ignore_index=True)
308324
tm.assert_series_equal(result, expected)
309325

310-
def test_concat_tz_series3(self):
326+
def test_concat_tz_series3(self, unit, unit2):
311327
# see gh-12217 and gh-12306
312328
# Concatenating two UTC times
313-
first = DataFrame([[datetime(2016, 1, 1)]])
329+
first = DataFrame([[datetime(2016, 1, 1)]], dtype=f"M8[{unit}]")
314330
first[0] = first[0].dt.tz_localize("UTC")
315331

316-
second = DataFrame([[datetime(2016, 1, 2)]])
332+
second = DataFrame([[datetime(2016, 1, 2)]], dtype=f"M8[{unit2}]")
317333
second[0] = second[0].dt.tz_localize("UTC")
318334

319335
result = concat([first, second])
320-
assert result[0].dtype == "datetime64[ns, UTC]"
336+
exp_unit = _get_finer_unit(unit, unit2)
337+
assert result[0].dtype == f"datetime64[{exp_unit}, UTC]"
321338

322-
def test_concat_tz_series4(self):
339+
def test_concat_tz_series4(self, unit, unit2):
323340
# Concatenating two London times
324-
first = DataFrame([[datetime(2016, 1, 1)]])
341+
first = DataFrame([[datetime(2016, 1, 1)]], dtype=f"M8[{unit}]")
325342
first[0] = first[0].dt.tz_localize("Europe/London")
326343

327-
second = DataFrame([[datetime(2016, 1, 2)]])
344+
second = DataFrame([[datetime(2016, 1, 2)]], dtype=f"M8[{unit2}]")
328345
second[0] = second[0].dt.tz_localize("Europe/London")
329346

330347
result = concat([first, second])
331-
assert result[0].dtype == "datetime64[ns, Europe/London]"
348+
exp_unit = _get_finer_unit(unit, unit2)
349+
assert result[0].dtype == f"datetime64[{exp_unit}, Europe/London]"
332350

333-
def test_concat_tz_series5(self):
351+
def test_concat_tz_series5(self, unit, unit2):
334352
# Concatenating 2+1 London times
335-
first = DataFrame([[datetime(2016, 1, 1)], [datetime(2016, 1, 2)]])
353+
first = DataFrame(
354+
[[datetime(2016, 1, 1)], [datetime(2016, 1, 2)]], dtype=f"M8[{unit}]"
355+
)
336356
first[0] = first[0].dt.tz_localize("Europe/London")
337357

338-
second = DataFrame([[datetime(2016, 1, 3)]])
358+
second = DataFrame([[datetime(2016, 1, 3)]], dtype=f"M8[{unit2}]")
339359
second[0] = second[0].dt.tz_localize("Europe/London")
340360

341361
result = concat([first, second])
342-
assert result[0].dtype == "datetime64[ns, Europe/London]"
362+
exp_unit = _get_finer_unit(unit, unit2)
363+
assert result[0].dtype == f"datetime64[{exp_unit}, Europe/London]"
343364

344-
def test_concat_tz_series6(self):
345-
# Concat'ing 1+2 London times
346-
first = DataFrame([[datetime(2016, 1, 1)]])
365+
def test_concat_tz_series6(self, unit, unit2):
366+
# Concatenating 1+2 London times
367+
first = DataFrame([[datetime(2016, 1, 1)]], dtype=f"M8[{unit}]")
347368
first[0] = first[0].dt.tz_localize("Europe/London")
348369

349-
second = DataFrame([[datetime(2016, 1, 2)], [datetime(2016, 1, 3)]])
370+
second = DataFrame(
371+
[[datetime(2016, 1, 2)], [datetime(2016, 1, 3)]], dtype=f"M8[{unit2}]"
372+
)
350373
second[0] = second[0].dt.tz_localize("Europe/London")
351374

352375
result = concat([first, second])
353-
assert result[0].dtype == "datetime64[ns, Europe/London]"
376+
exp_unit = _get_finer_unit(unit, unit2)
377+
assert result[0].dtype == f"datetime64[{exp_unit}, Europe/London]"
354378

355379
def test_concat_tz_series_tzlocal(self):
356380
# see gh-13583

0 commit comments

Comments
 (0)