diff --git a/doc/source/release.rst b/doc/source/release.rst index 95ce03a858570..0eafd33d58521 100644 --- a/doc/source/release.rst +++ b/doc/source/release.rst @@ -135,6 +135,8 @@ pandas 0.13 - Raise on set indexing with a Panel and a Panel as a value which needs alignment (:issue:`3777`) - frozenset objects now raise in the ``Series`` constructor (:issue:`4482`, :issue:`4480`) + - Fixed bug where timedelta and timedelta64 were treated differently when + being add to a Series (:issue:`4135`) pandas 0.12 =========== @@ -461,6 +463,7 @@ pandas 0.12 - Fixed bug where get_data_famafrench wasn't using the correct file edges (:issue:`4281`) + pandas 0.11.0 ============= diff --git a/doc/source/v0.13.0.txt b/doc/source/v0.13.0.txt index 7da2f03ad4c74..3c351bad4a1bc 100644 --- a/doc/source/v0.13.0.txt +++ b/doc/source/v0.13.0.txt @@ -110,6 +110,9 @@ Bug Fixes - Suppressed DeprecationWarning associated with internal calls issued by repr() (:issue:`4391`) + - Fixed bug where timedelta and timedelta64 were treated differently when + being add to a Series (:issue:`4135`) + See the :ref:`full release notes ` or issue tracker on GitHub for a complete list. diff --git a/pandas/core/common.py b/pandas/core/common.py index 06ca3be455f2a..c9d62b53a9d74 100644 --- a/pandas/core/common.py +++ b/pandas/core/common.py @@ -3,8 +3,10 @@ """ import re +from datetime import timedelta import codecs import csv +from distutils.version import LooseVersion from numpy.lib.format import read_array, write_array import numpy as np @@ -14,12 +16,16 @@ import pandas.tslib as tslib from pandas import compat -from pandas.compat import StringIO, BytesIO, range, long, u, zip, map +from pandas.compat import StringIO, BytesIO, range, long, u, zip, map, lmap from pandas.core.config import get_option from pandas.core import array as pa +_np_version = np.version.short_version +_np_version_under1p6 = LooseVersion(_np_version) < '1.6' +_np_version_under1p7 = LooseVersion(_np_version) < '1.7' + # XXX: HACK for NumPy 1.5.1 to suppress warnings try: np.seterr(all='ignore') @@ -293,10 +299,10 @@ def _take_2d_multi_generic(arr, indexer, out, fill_value, mask_info): if col_needs: out[:, col_mask] = fill_value for i in range(len(row_idx)): - u = row_idx[i] + _u = row_idx[i] for j in range(len(col_idx)): v = col_idx[j] - out[i, j] = arr[u, v] + out[i, j] = arr[_u, v] def _take_nd_generic(arr, indexer, out, axis, fill_value, mask_info): @@ -1099,6 +1105,18 @@ def _consensus_name_attr(objs): # Lots of little utilities +def _td_to_us(td): + d = td.days + s = td.seconds + us = td.microseconds + v = us + (s + d * 24 * 3600) * 10 ** 6 + return timedelta(microseconds=v) + + +def _td_array_to_us(a): + return np.asanyarray(lmap(lambda x: x.item(), a)) + + def _possibly_convert_objects(values, convert_dates=True, convert_numeric=True): """ if we have an object dtype, try to coerce dates and/or numers """ @@ -1149,8 +1167,10 @@ def _possibly_cast_to_timedelta(value, coerce=True): don't force the conversion unless coerce is True """ # deal with numpy not being able to handle certain timedelta operations - if isinstance(value,np.ndarray) and value.dtype.kind == 'm': + if isinstance(value, np.ndarray) and value.dtype.kind == 'm': if value.dtype != 'timedelta64[ns]': + if _np_version_under1p7: + value = _td_array_to_us(value) value = value.astype('timedelta64[ns]') return value diff --git a/pandas/core/series.py b/pandas/core/series.py index 58fd0a0551ace..63cf2a16b593d 100644 --- a/pandas/core/series.py +++ b/pandas/core/series.py @@ -47,9 +47,6 @@ __all__ = ['Series', 'TimeSeries'] -_np_version = np.version.short_version -_np_version_under1p6 = LooseVersion(_np_version) < '1.6' -_np_version_under1p7 = LooseVersion(_np_version) < '1.7' _SHOW_WARNINGS = True @@ -104,8 +101,7 @@ def convert_to_array(values): elif inferred_type in set(['timedelta','timedelta64']): # need to convert timedelta to ns here # safest to convert it to an object arrany to process - if not (isinstance(values, pa.Array) and com.is_timedelta64_dtype(values)): - values = com._possibly_cast_to_timedelta(values) + values = com._possibly_cast_to_timedelta(values) elif inferred_type in set(['integer']): if values.dtype.kind == 'm': values = values.astype('timedelta64[ns]') @@ -126,7 +122,7 @@ def convert_to_array(values): if is_datetime_lhs and name != '__sub__': raise TypeError("can only operate on a datetimes for subtraction, " "but the operator [%s] was passed" % name) - elif is_timedelta_lhs and name not in ['__add__','__sub__']: + elif is_timedelta_lhs and name not in ['__add__', '__sub__']: raise TypeError("can only operate on a timedeltas for " "addition and subtraction, but the operator [%s] was passed" % name) @@ -143,13 +139,13 @@ def wrap_results(x): # datetime and timedelta elif (is_timedelta_lhs and is_datetime_rhs) or (is_timedelta_rhs and is_datetime_lhs): - if name not in ['__add__','__sub__']: + if name not in ['__add__', '__sub__']: raise TypeError("can only operate on a timedelta and a datetime for " "addition and subtraction, but the operator [%s] was passed" % name) dtype = 'M8[ns]' else: - raise ValueError('cannot operate on a series with out a rhs ' + raise ValueError('cannot operate on a series without a rhs ' 'of a series/ndarray of type datetime64[ns] ' 'or a timedelta') @@ -291,8 +287,7 @@ def _radd_compat(left, right): try: output = radd(left, right) except TypeError: - cond = (_np_version_under1p6 and - left.dtype == np.object_) + cond = com._np_version_under1p6 and left.dtype == np.object_ if cond: # pragma: no cover output = np.empty_like(left) output.flat[:] = [radd(x, right) for x in left.flat] @@ -806,8 +801,13 @@ def abs(self): abs: type of caller """ obj = np.abs(self) - obj = com._possibly_cast_to_timedelta(obj, coerce=False) - return obj + return self._constructor(obj, name=self.name) + + def __array_wrap__(self, out, ctx=None): + if (com._np_version_under1p7 and out.dtype.kind == 'm' and + out.dtype != _TD_DTYPE): + out = out.view('i8').astype(_TD_DTYPE) + return pa.Array.__array_wrap__(self, out, ctx) def __setitem__(self, key, value): try: diff --git a/pandas/tests/test_series.py b/pandas/tests/test_series.py index b192aded45074..da61e9a51ff53 100644 --- a/pandas/tests/test_series.py +++ b/pandas/tests/test_series.py @@ -1,10 +1,10 @@ # pylint: disable-msg=E1101,W0612 -from datetime import datetime, timedelta, date -import os +from datetime import datetime, timedelta import operator import unittest import string +from itertools import starmap, product import nose @@ -21,6 +21,7 @@ import pandas.core.series as smod import pandas.lib as lib +from pandas.core import common as com import pandas.core.datetools as datetools import pandas.core.nanops as nanops @@ -1393,7 +1394,6 @@ def test_timeseries_periodindex(self): new_ts = pickle.loads(pickle.dumps(ts)) self.assertEqual(new_ts.index.freq,'M') - def test_iter(self): for i, val in enumerate(self.series): self.assertEqual(val, self.series[i]) @@ -2019,7 +2019,8 @@ def test_timedelta64_functions(self): #result = np.abs(s1-s2) #assert_frame_equal(result,expected) - result = (s1-s2).abs() + d = s1 - s2 + result = d.abs() assert_series_equal(result,expected) # max/min @@ -2031,6 +2032,32 @@ def test_timedelta64_functions(self): expected = Series([timedelta(1)],dtype='timedelta64[ns]') assert_series_equal(result,expected) + def test_timedelta64_equal_timedelta_supported_ops(self): + ser = Series([Timestamp('20130301'), Timestamp('20130228 23:00:00'), + Timestamp('20130228 22:00:00'), + Timestamp('20130228 21:00:00')]) + + intervals = 'D', 'h', 'm', 's', 'us' + npy16_mappings = {'D': 24 * 60 * 60 * 1000000, 'h': 60 * 60 * 1000000, + 'm': 60 * 1000000, 's': 1000000, 'us': 1} + + def timedelta64(*args): + if com._np_version_under1p7: + coeffs = np.array(args) + terms = np.array([npy16_mappings[interval] + for interval in intervals]) + return np.timedelta64(coeffs.dot(terms)) + return sum(starmap(np.timedelta64, zip(args, intervals))) + + for op, d, h, m, s, us in product([operator.add, operator.sub], + *([range(2)] * 5)): + nptd = timedelta64(d, h, m, s, us) + pytd = timedelta(days=d, hours=h, minutes=m, seconds=s, + microseconds=us) + lhs = op(ser, nptd) + rhs = op(ser, pytd) + + assert_series_equal(lhs, rhs) def test_sub_of_datetime_from_TimeSeries(self): from pandas.core import common as com @@ -3210,7 +3237,7 @@ def test_getitem_setitem_datetime_tz(self): assert_series_equal(result, ts) def test_getitem_setitem_periodindex(self): - from pandas import period_range, Period + from pandas import period_range N = 50 rng = period_range('1/1/1990', periods=N, freq='H') ts = Series(np.random.randn(N), index=rng)