Skip to content

Commit ff1f0f2

Browse files
jrebackjreback
committed
fix datetime block evals
1 parent 2a8f11f commit ff1f0f2

File tree

6 files changed

+74
-43
lines changed

6 files changed

+74
-43
lines changed

‎pandas/core/frame.py

Lines changed: 29 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -3570,7 +3570,8 @@ def reorder_levels(self, order, axis=0):
35703570
# ----------------------------------------------------------------------
35713571
# Arithmetic / combination related
35723572

3573-
def _combine_frame(self, other, func, fill_value=None, level=None):
3573+
def _combine_frame(self, other, func, fill_value=None, level=None,
3574+
try_cast=True):
35743575
this, other = self.align(other, join='outer', level=level, copy=False)
35753576
new_index, new_columns = this.index, this.columns
35763577

@@ -3623,19 +3624,23 @@ def f(i):
36233624
copy=False)
36243625

36253626
def _combine_series(self, other, func, fill_value=None, axis=None,
3626-
level=None):
3627+
level=None, try_cast=True):
36273628
if axis is not None:
36283629
axis = self._get_axis_name(axis)
36293630
if axis == 'index':
36303631
return self._combine_match_index(other, func, level=level,
3631-
fill_value=fill_value)
3632+
fill_value=fill_value,
3633+
try_cast=try_cast)
36323634
else:
36333635
return self._combine_match_columns(other, func, level=level,
3634-
fill_value=fill_value)
3636+
fill_value=fill_value,
3637+
try_cast=try_cast)
36353638
return self._combine_series_infer(other, func, level=level,
3636-
fill_value=fill_value)
3639+
fill_value=fill_value,
3640+
try_cast=try_cast)
36373641

3638-
def _combine_series_infer(self, other, func, level=None, fill_value=None):
3642+
def _combine_series_infer(self, other, func, level=None,
3643+
fill_value=None, try_cast=True):
36393644
if len(other) == 0:
36403645
return self * NA
36413646

@@ -3645,9 +3650,11 @@ def _combine_series_infer(self, other, func, level=None, fill_value=None):
36453650
columns=self.columns)
36463651

36473652
return self._combine_match_columns(other, func, level=level,
3648-
fill_value=fill_value)
3653+
fill_value=fill_value,
3654+
try_cast=try_cast)
36493655

3650-
def _combine_match_index(self, other, func, level=None, fill_value=None):
3656+
def _combine_match_index(self, other, func, level=None,
3657+
fill_value=None, try_cast=True):
36513658
left, right = self.align(other, join='outer', axis=0, level=level,
36523659
copy=False)
36533660
if fill_value is not None:
@@ -3657,23 +3664,26 @@ def _combine_match_index(self, other, func, level=None, fill_value=None):
36573664
index=left.index, columns=self.columns,
36583665
copy=False)
36593666

3660-
def _combine_match_columns(self, other, func, level=None, fill_value=None):
3667+
def _combine_match_columns(self, other, func, level=None,
3668+
fill_value=None, try_cast=True):
36613669
left, right = self.align(other, join='outer', axis=1, level=level,
36623670
copy=False)
36633671
if fill_value is not None:
36643672
raise NotImplementedError("fill_value %r not supported" %
36653673
fill_value)
36663674

36673675
new_data = left._data.eval(func=func, other=right,
3668-
axes=[left.columns, self.index])
3676+
axes=[left.columns, self.index],
3677+
try_cast=try_cast)
36693678
return self._constructor(new_data)
36703679

3671-
def _combine_const(self, other, func, raise_on_error=True):
3680+
def _combine_const(self, other, func, raise_on_error=True, try_cast=True):
36723681
new_data = self._data.eval(func=func, other=other,
3673-
raise_on_error=raise_on_error)
3682+
raise_on_error=raise_on_error,
3683+
try_cast=try_cast)
36743684
return self._constructor(new_data)
36753685

3676-
def _compare_frame_evaluate(self, other, func, str_rep):
3686+
def _compare_frame_evaluate(self, other, func, str_rep, try_cast=True):
36773687

36783688
# unique
36793689
if self.columns.is_unique:
@@ -3697,16 +3707,18 @@ def _compare(a, b):
36973707
result.columns = self.columns
36983708
return result
36993709

3700-
def _compare_frame(self, other, func, str_rep):
3710+
def _compare_frame(self, other, func, str_rep, try_cast=True):
37013711
if not self._indexed_same(other):
37023712
raise ValueError('Can only compare identically-labeled '
37033713
'DataFrame objects')
3704-
return self._compare_frame_evaluate(other, func, str_rep)
3714+
return self._compare_frame_evaluate(other, func, str_rep,
3715+
try_cast=try_cast)
37053716

3706-
def _flex_compare_frame(self, other, func, str_rep, level):
3717+
def _flex_compare_frame(self, other, func, str_rep, level, try_cast=True):
37073718
if not self._indexed_same(other):
37083719
self, other = self.align(other, 'outer', level=level, copy=False)
3709-
return self._compare_frame_evaluate(other, func, str_rep)
3720+
return self._compare_frame_evaluate(other, func, str_rep,
3721+
try_cast=try_cast)
37103722

37113723
def combine(self, other, func, fill_value=None, overwrite=True):
37123724
"""

‎pandas/core/internals.py

Lines changed: 18 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1029,19 +1029,27 @@ def coerce_to_target_dtype(self, other):
10291029
# we don't upcast to bool
10301030
return self.astype(object)
10311031

1032-
elif self.is_datelike:
1032+
elif (self.is_datetime or
1033+
is_datetime64_dtype(dtype) or
1034+
is_datetime64tz_dtype(dtype)):
10331035

1034-
# we don't upcast i8
1035-
if is_integer_dtype(dtype):
1036+
# not a datetime
1037+
if not ((is_datetime64_dtype(dtype) or
1038+
is_datetime64tz_dtype(dtype)) and self.is_datetime):
10361039
return self.astype(object)
10371040

10381041
# don't upcast timezone with different timezone or no timezone
1039-
if self.is_datetime:
1040-
mytz = getattr(self.dtype, 'tz', None)
1041-
othertz = getattr(dtype, 'tz', None)
1042+
mytz = getattr(self.dtype, 'tz', None)
1043+
othertz = getattr(dtype, 'tz', None)
10421044

1043-
if str(mytz) != str(othertz):
1044-
return self.astype(object)
1045+
if str(mytz) != str(othertz):
1046+
return self.astype(object)
1047+
1048+
elif (self.is_timedelta or is_timedelta64_dtype(dtype)):
1049+
1050+
# not a timedelta
1051+
if not (is_timedelta64_dtype(dtype) and self.is_timedelta):
1052+
return self.astype(object)
10451053

10461054
try:
10471055
return self.astype(dtype)
@@ -1315,7 +1323,7 @@ def get_result(other):
13151323
result = result.astype('float64', copy=False)
13161324
result[other_mask.ravel()] = np.nan
13171325

1318-
return self._try_coerce_result(result)
1326+
return result
13191327

13201328
# error handler if we have an issue operating with the function
13211329
def handle_error():
@@ -1363,6 +1371,7 @@ def handle_error():
13631371
if try_cast:
13641372
result = self._try_cast_result(result)
13651373

1374+
result = _block_shape(result, ndim=self.ndim)
13661375
return [self.make_block(result, fastpath=True, )]
13671376

13681377
def where(self, other, cond, align=True, raise_on_error=True,

‎pandas/core/ops.py

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1278,12 +1278,14 @@ def f(self, other, axis=default_axis, level=None):
12781278
other = _align_method_FRAME(self, other, axis)
12791279

12801280
if isinstance(other, pd.DataFrame): # Another DataFrame
1281-
return self._flex_compare_frame(other, na_op, str_rep, level)
1281+
return self._flex_compare_frame(other, na_op, str_rep, level,
1282+
try_cast=False)
12821283

12831284
elif isinstance(other, ABCSeries):
1284-
return self._combine_series(other, na_op, None, axis, level)
1285+
return self._combine_series(other, na_op, None, axis, level,
1286+
try_cast=False)
12851287
else:
1286-
return self._combine_const(other, na_op)
1288+
return self._combine_const(other, na_op, try_cast=False)
12871289

12881290
f.__name__ = name
12891291

@@ -1296,12 +1298,14 @@ def f(self, other):
12961298
if isinstance(other, pd.DataFrame): # Another DataFrame
12971299
return self._compare_frame(other, func, str_rep)
12981300
elif isinstance(other, ABCSeries):
1299-
return self._combine_series_infer(other, func)
1301+
return self._combine_series_infer(other, func, try_cast=False)
13001302
else:
13011303

13021304
# straight boolean comparisions we want to allow all columns
13031305
# (regardless of dtype to pass thru) See #4537 for discussion.
1304-
res = self._combine_const(other, func, raise_on_error=False)
1306+
res = self._combine_const(other, func,
1307+
raise_on_error=False,
1308+
try_cast=False)
13051309
return res.fillna(True).astype(bool)
13061310

13071311
f.__name__ = name
@@ -1381,13 +1385,13 @@ def f(self, other, axis=None):
13811385
axis = self._get_axis_number(axis)
13821386

13831387
if isinstance(other, self._constructor):
1384-
return self._compare_constructor(other, na_op)
1388+
return self._compare_constructor(other, na_op, try_cast=False)
13851389
elif isinstance(other, (self._constructor_sliced, pd.DataFrame,
13861390
ABCSeries)):
13871391
raise Exception("input needs alignment for this object [%s]" %
13881392
self._constructor)
13891393
else:
1390-
return self._combine_const(other, na_op)
1394+
return self._combine_const(other, na_op, try_cast=False)
13911395

13921396
f.__name__ = name
13931397

‎pandas/core/panel.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -324,7 +324,7 @@ def _init_matrix(self, data, axes, dtype=None, copy=False):
324324
# ----------------------------------------------------------------------
325325
# Comparison methods
326326

327-
def _compare_constructor(self, other, func):
327+
def _compare_constructor(self, other, func, try_cast=True):
328328
if not self._indexed_same(other):
329329
raise Exception('Can only compare identically-labeled '
330330
'same type objects')
@@ -714,13 +714,13 @@ def _combine(self, other, func, axis=0):
714714
"operation with %s" %
715715
(str(type(other)), str(type(self))))
716716

717-
def _combine_const(self, other, func):
717+
def _combine_const(self, other, func, try_cast=True):
718718
with np.errstate(all='ignore'):
719719
new_values = func(self.values, other)
720720
d = self._construct_axes_dict()
721721
return self._constructor(new_values, **d)
722722

723-
def _combine_frame(self, other, func, axis=0):
723+
def _combine_frame(self, other, func, axis=0, try_cast=True):
724724
index, columns = self._get_plane_axes(axis)
725725
axis = self._get_axis_number(axis)
726726

@@ -739,7 +739,7 @@ def _combine_frame(self, other, func, axis=0):
739739
return self._constructor(new_values, self.items, self.major_axis,
740740
self.minor_axis)
741741

742-
def _combine_panel(self, other, func):
742+
def _combine_panel(self, other, func, try_cast=True):
743743
items = self.items.union(other.items)
744744
major = self.major_axis.union(other.major_axis)
745745
minor = self.minor_axis.union(other.minor_axis)

‎pandas/core/sparse/frame.py

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -501,7 +501,8 @@ def xs(self, key, axis=0, copy=False):
501501
# ----------------------------------------------------------------------
502502
# Arithmetic-related methods
503503

504-
def _combine_frame(self, other, func, fill_value=None, level=None):
504+
def _combine_frame(self, other, func, fill_value=None, level=None,
505+
try_cast=True):
505506
this, other = self.align(other, join='outer', level=level, copy=False)
506507
new_index, new_columns = this.index, this.columns
507508

@@ -544,7 +545,8 @@ def _combine_frame(self, other, func, fill_value=None, level=None):
544545
default_fill_value=new_fill_value
545546
).__finalize__(self)
546547

547-
def _combine_match_index(self, other, func, level=None, fill_value=None):
548+
def _combine_match_index(self, other, func, level=None, fill_value=None,
549+
try_cast=True):
548550
new_data = {}
549551

550552
if fill_value is not None:
@@ -574,7 +576,8 @@ def _combine_match_index(self, other, func, level=None, fill_value=None):
574576
new_data, index=new_index, columns=self.columns,
575577
default_fill_value=fill_value).__finalize__(self)
576578

577-
def _combine_match_columns(self, other, func, level=None, fill_value=None):
579+
def _combine_match_columns(self, other, func, level=None, fill_value=None,
580+
try_cast=True):
578581
# patched version of DataFrame._combine_match_columns to account for
579582
# NumPy circumventing __rsub__ with float64 types, e.g.: 3.0 - series,
580583
# where 3.0 is numpy.float64 and series is a SparseSeries. Still
@@ -600,7 +603,7 @@ def _combine_match_columns(self, other, func, level=None, fill_value=None):
600603
new_data, index=self.index, columns=union,
601604
default_fill_value=self.default_fill_value).__finalize__(self)
602605

603-
def _combine_const(self, other, func, raise_on_error=True):
606+
def _combine_const(self, other, func, raise_on_error=True, try_cast=True):
604607
return self._apply_columns(lambda x: func(x, other))
605608

606609
def _reindex_index(self, index, method, copy, level, fill_value=np.nan,

‎pandas/tests/frame/test_operators.py

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,7 @@ def test_timestamp_compare(self):
188188
df.loc[np.random.rand(len(df)) > 0.5, 'dates2'] = pd.NaT
189189
ops = {'gt': 'lt', 'lt': 'gt', 'ge': 'le', 'le': 'ge', 'eq': 'eq',
190190
'ne': 'ne'}
191+
191192
for left, right in ops.items():
192193
left_f = getattr(operator, left)
193194
right_f = getattr(operator, right)
@@ -832,9 +833,11 @@ def test_combineSeries(self):
832833
assert 'E' in larger_added
833834
assert np.isnan(larger_added['E']).all()
834835

835-
# vs mix (upcast) as needed
836+
# no upcast needed
836837
added = self.mixed_float + series
837-
_check_mixed_float(added, dtype='float64')
838+
_check_mixed_float(added)
839+
840+
# vs mix (upcast) as needed
838841
added = self.mixed_float + series.astype('float32')
839842
_check_mixed_float(added, dtype=dict(C=None))
840843
added = self.mixed_float + series.astype('float16')

0 commit comments

Comments
 (0)