Skip to content

Commit d99c9bd

Browse files
committed
BUG: Handle Datetimelike data in DataFrame.combine
Closes pandas-dev#23079
1 parent 92fd46c commit d99c9bd

File tree

3 files changed

+42
-19
lines changed

3 files changed

+42
-19
lines changed

doc/source/whatsnew/v0.24.0.txt

+1
Original file line numberDiff line numberDiff line change
@@ -1000,6 +1000,7 @@ Datetimelike
10001000
- Bug in :func:`to_datetime` with an :class:`Index` argument that would drop the ``name`` from the result (:issue:`21697`)
10011001
- Bug in :class:`PeriodIndex` where adding or subtracting a :class:`timedelta` or :class:`Tick` object produced incorrect results (:issue:`22988`)
10021002
- Bug in :func:`date_range` when decrementing a start date to a past end date by a negative frequency (:issue:`23270`)
1003+
- Bug in :func:`DataFrame.combine` with datetimelike values raising a TypeError (:issue:`23079`)
10031004

10041005
Timedelta
10051006
^^^^^^^^^

pandas/core/frame.py

+23-19
Original file line numberDiff line numberDiff line change
@@ -5141,22 +5141,14 @@ def combine(self, other, func, fill_value=None, overwrite=True):
51415141
if not is_dtype_equal(other_dtype, new_dtype):
51425142
otherSeries = otherSeries.astype(new_dtype)
51435143

5144-
# see if we need to be represented as i8 (datetimelike)
5145-
# try to keep us at this dtype
5146-
needs_i8_conversion_i = needs_i8_conversion(new_dtype)
5147-
if needs_i8_conversion_i:
5148-
arr = func(series, otherSeries, True)
5149-
else:
5150-
arr = func(series, otherSeries)
5151-
5144+
arr = func(series, otherSeries)
51525145
arr = maybe_downcast_to_dtype(arr, this_dtype)
51535146

51545147
result[col] = arr
51555148

51565149
# convert_objects just in case
51575150
return self._constructor(result, index=new_index,
5158-
columns=new_columns)._convert(datetime=True,
5159-
copy=False)
5151+
columns=new_columns)
51605152

51615153
def combine_first(self, other):
51625154
"""
@@ -5203,15 +5195,27 @@ def combine_first(self, other):
52035195
"""
52045196
import pandas.core.computation.expressions as expressions
52055197

5206-
def combiner(x, y, needs_i8_conversion=False):
5207-
x_values = x.values if hasattr(x, 'values') else x
5208-
y_values = y.values if hasattr(y, 'values') else y
5209-
if needs_i8_conversion:
5210-
mask = isna(x)
5211-
x_values = x_values.view('i8')
5212-
y_values = y_values.view('i8')
5213-
else:
5214-
mask = isna(x_values)
5198+
def extract_values(arr):
5199+
# Does two things
5200+
# maybe gets the values from the Series / Index
5201+
# convert datelike to i8
5202+
if isinstance(arr, (ABCIndexClass, ABCSeries)):
5203+
arr = arr._values
5204+
5205+
if needs_i8_conversion(arr):
5206+
if is_extension_array_dtype(arr.dtype):
5207+
arr = arr.asi8
5208+
else:
5209+
arr = arr.view('i8')
5210+
return arr
5211+
5212+
def combiner(x, y):
5213+
mask = isna(x)
5214+
if isinstance(mask, (ABCIndexClass, ABCSeries)):
5215+
mask = mask.values
5216+
5217+
x_values = extract_values(x)
5218+
y_values = extract_values(y)
52155219

52165220
# If the column y in other DataFrame is not in first DataFrame,
52175221
# just return y_values.

pandas/tests/frame/test_combine_concat.py

+18
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,24 @@ def test_concat_multiple_frames_dtypes(self):
3131
expected = Series(dict(float64=2, float32=2))
3232
assert_series_equal(results, expected)
3333

34+
@pytest.mark.parametrize('data', [
35+
pd.date_range('2000', periods=4),
36+
pd.date_range('2000', periods=4, tz="US/Central"),
37+
pd.period_range('2000', periods=4),
38+
pd.timedelta_range(0, periods=4),
39+
])
40+
def test_combine_datetlike_udf(self, data):
41+
# https://github.com/pandas-dev/pandas/issues/23079
42+
df = pd.DataFrame({"A": data})
43+
other = df.copy()
44+
df.iloc[1, 0] = None
45+
46+
def combiner(a, b):
47+
return b
48+
49+
result = df.combine(other, combiner)
50+
tm.assert_frame_equal(result, other)
51+
3452
def test_concat_multiple_tzs(self):
3553
# GH 12467
3654
# combining datetime tz-aware and naive DataFrames

0 commit comments

Comments
 (0)