DOC: release note for timedelta changes

TST: skip tests on numpy < 1.7 as needed for timedeltas

BUG: fixup .abs for timedeltas

Author: jreback

doc/source/release.rst

@@ -62,6 +62,8 @@ API Changes
     when detecting chained assignment, related (:issue:`5938`)
   - DataFrame.head(0) returns self instead of empty frame (:issue:`5846`)
   - ``autocorrelation_plot`` now accepts ``**kwargs``. (:issue:`5623`)
+  - ``convert_objects`` now accepts a ``convert_timedeltas='coerce'`` argument to allow forced dtype conversion of
+    timedeltas (:issue:`5458`,:issue:`5689`)
 
 Experimental Features
 ~~~~~~~~~~~~~~~~~~~~~
@@ -78,12 +80,13 @@ Improvements to existing features
   - support ``dtypes`` property on ``Series/Panel/Panel4D``
   - extend ``Panel.apply`` to allow arbitrary functions (rather than only ufuncs) (:issue:`1148`)
     allow multiple axes to be used to operate on slabs of a ``Panel``
-  - The ``ArrayFormatter``s for ``datetime`` and ``timedelta64`` now intelligently
+  - The ``ArrayFormatter`` for ``datetime`` and ``timedelta64`` now intelligently
     limit precision based on the values in the array (:issue:`3401`)
   - pd.show_versions() is now available for convenience when reporting issues.
   - perf improvements to Series.str.extract (:issue:`5944`)
   - perf improvments in ``dtypes/ftypes`` methods (:issue:`5968`)
   - perf improvments in indexing with object dtypes (:issue:`5968`)
+  - improved dtype inference for ``timedelta`` like passed to constructors (:issue:`5458`,:issue:`5689`)
 
 .. _release.bug_fixes-0.13.1:
 
@@ -122,6 +125,7 @@ Bug Fixes
   - Recent changes in IPython cause warnings to be emitted when using previous versions
     of pandas in QTConsole, now fixed. If you're using an older version and
     need to supress the warnings, see (:issue:`5922`).
+  - Bug in merging ``timedelta`` dtypes (:issue:`5695`)
 
 pandas 0.13.0
 -------------

doc/source/v0.13.1.txt

@@ -3,7 +3,7 @@
 v0.13.1 (???)
 -------------
 
-This is a major release from 0.13.0 and includes a number of API changes, several new features and
+This is a minor release from 0.13.0 and includes a number of API changes, several new features and
 enhancements along with a large number of bug fixes.
 
 Highlights include:
@@ -29,6 +29,27 @@ Deprecations
 Enhancements
 ~~~~~~~~~~~~
 
+- The ``ArrayFormatter`` for ``datetime`` and ``timedelta64`` now intelligently
+  limit precision based on the values in the array (:issue:`3401`)
+
+  Previously output might look like:
+
+  .. code-block:: python
+
+        age                 today               diff
+      0 2001-01-01 00:00:00 2013-04-19 00:00:00 4491 days, 00:00:00
+      1 2004-06-01 00:00:00 2013-04-19 00:00:00 3244 days, 00:00:00
+
+  Now the output looks like:
+
+   .. ipython:: python
+
+      df = DataFrame([ Timestamp('20010101'),
+                       Timestamp('20040601') ], columns=['age'])
+      df['today'] = Timestamp('20130419')
+      df['diff'] = df['today']-df['age']
+      df
+
 - ``Panel.apply`` will work on non-ufuncs. See :ref:`the docs<basics.apply_panel>`.
 
   .. ipython:: python
@@ -83,27 +104,6 @@ Enhancements
      result
      result.loc[:,:,'ItemA']
 
-- The ``ArrayFormatter``s for ``datetime`` and ``timedelta64`` now intelligently
-  limit precision based on the values in the array (:issue:`3401`)
-  
-  Previously output might look like:
-
-  .. code-block:: python
-   
-        age                 today               diff
-      0 2001-01-01 00:00:00 2013-04-19 00:00:00 4491 days, 00:00:00
-      1 2004-06-01 00:00:00 2013-04-19 00:00:00 3244 days, 00:00:00
-      
-  Now the output looks like:
-
-   .. ipython:: python
-   
-      df = DataFrame([ Timestamp('20010101'),
-                       Timestamp('20040601') ], columns=['age'])
-      df['today'] = Timestamp('20130419')
-      df['diff'] = df['today']-df['age']
-      df
-   
 Experimental
 ~~~~~~~~~~~~
 

pandas/core/common.py

@@ -1641,7 +1641,7 @@ def _possibly_cast_to_datetime(value, dtype, coerce=False):
                         elif is_timedelta64:
                             from pandas.tseries.timedeltas import \
                                 _possibly_cast_to_timedelta
-                            value = _possibly_cast_to_timedelta(value, coerce=True)
+                            value = _possibly_cast_to_timedelta(value, coerce='compat')
                     except:
                         pass
 
@@ -1672,7 +1672,7 @@ def _possibly_cast_to_datetime(value, dtype, coerce=False):
                 elif inferred_type in ['timedelta', 'timedelta64']:
                     from pandas.tseries.timedeltas import \
                         _possibly_cast_to_timedelta
-                    value = _possibly_cast_to_timedelta(value)
+                    value = _possibly_cast_to_timedelta(value, coerce='compat')
 
     return value
 

pandas/core/generic.py

@@ -3177,23 +3177,22 @@ def abs(self):
         -------
         abs: type of caller
         """
-        obj = np.abs(self)
 
         # suprimo numpy 1.6 hacking
+        # for timedeltas
         if _np_version_under1p7:
+
+            def _convert_timedeltas(x):
+                if x.dtype.kind == 'm':
+                    return np.abs(x.view('i8')).astype(x.dtype)
+                return np.abs(x)
+
             if self.ndim == 1:
-                if obj.dtype == 'm8[us]':
-                    obj = obj.astype('m8[ns]')
+                return _convert_timedeltas(self)
             elif self.ndim == 2:
-                def f(x):
-                    if x.dtype == 'm8[us]':
-                        x = x.astype('m8[ns]')
-                    return x
+                return  self.apply(_convert_timedeltas)
 
-                if 'm8[us]' in obj.dtypes.values:
-                    obj = obj.apply(f)
-
-        return obj
+        return np.abs(self)
 
     def pct_change(self, periods=1, fill_method='pad', limit=None, freq=None,
                    **kwds):

pandas/lib.pyx

@@ -14,7 +14,7 @@ from cpython cimport (PyDict_New, PyDict_GetItem, PyDict_SetItem,
                       Py_INCREF, PyTuple_SET_ITEM,
                       PyList_Check, PyFloat_Check,
                       PyString_Check,
-		      PyBytes_Check,
+                      PyBytes_Check,
                       PyTuple_SetItem,
                       PyTuple_New,
                       PyObject_SetAttrString)

pandas/src/datetime.pxd

@@ -37,6 +37,7 @@ cdef extern from "datetime.h":
     bint PyDateTime_Check(object o)
     bint PyDate_Check(object o)
     bint PyTime_Check(object o)
+    bint PyDelta_Check(object o)
     object PyDateTime_FromDateAndTime(int year, int month, int day, int hour,
                                       int minute, int second, int us)
 

pandas/src/inference.pyx

@@ -1,6 +1,6 @@
 cimport util
 from tslib import NaT
-
+from datetime import datetime, timedelta
 iNaT = util.get_nat()
 
 _TYPE_MAP = {
@@ -141,6 +141,9 @@ cdef inline bint is_date(object o):
 cdef inline bint is_time(object o):
     return PyTime_Check(o)
 
+cdef inline bint is_timedelta(object o):
+    return PyDelta_Check(o) or util.is_timedelta64_object(o)
+
 def is_bool_array(ndarray values):
     cdef:
         Py_ssize_t i, n = len(values)
@@ -296,19 +299,14 @@ def is_datetime64_array(ndarray values):
             return False
     return True
 
-def is_timedelta(object o):
-    import datetime
-    return isinstance(o,datetime.timedelta) or isinstance(o,np.timedelta64)
-
 def is_timedelta_array(ndarray values):
-    import datetime
     cdef int i, n = len(values)
     cdef object v
     if n == 0:
         return False
     for i in range(n):
         v = values[i]
-        if not (isinstance(v,datetime.timedelta) or is_null_datetimelike(v)):
+        if not (PyDelta_Check(v) or is_null_datetimelike(v)):
             return False
     return True
 
@@ -319,21 +317,19 @@ def is_timedelta64_array(ndarray values):
         return False
     for i in range(n):
         v = values[i]
-        if not (isinstance(v,np.timedelta64) or is_null_datetimelike(v)):
+        if not (util.is_timedelta64_object(v) or is_null_datetimelike(v)):
             return False
     return True
 
 def is_timedelta_or_timedelta64_array(ndarray values):
     """ infer with timedeltas and/or nat/none """
-    import datetime
     cdef int i, n = len(values)
     cdef object v
     if n == 0:
         return False
     for i in range(n):
         v = values[i]
-        if not (isinstance(v,datetime.timedelta) or isinstance(v,np.timedelta64) or
-                is_null_datetimelike(v)):
+        if not (is_timedelta(v) or is_null_datetimelike(v)):
             return False
     return True
 
@@ -486,10 +482,14 @@ def maybe_convert_objects(ndarray[object] objects, bint try_float=0,
     complexes = np.empty(n, dtype='c16')
     ints = np.empty(n, dtype='i8')
     bools = np.empty(n, dtype=np.uint8)
-    datetimes = np.empty(n, dtype='M8[ns]')
-    idatetimes = datetimes.view(np.int64)
-    timedeltas = np.empty(n, dtype='m8[ns]')
-    itimedeltas = timedeltas.view(np.int64)
+
+    if convert_datetime:
+        datetimes = np.empty(n, dtype='M8[ns]')
+        idatetimes = datetimes.view(np.int64)
+
+    if convert_timedelta:
+        timedeltas = np.empty(n, dtype='m8[ns]')
+        itimedeltas = timedeltas.view(np.int64)
 
     onan = np.nan
     fnan = np.nan
@@ -625,8 +625,6 @@ def try_parse_dates(ndarray[object] values, parser=None,
         Py_ssize_t i, n
         ndarray[object] result
 
-    from datetime import datetime, timedelta
-
     n = len(values)
     result = np.empty(n, dtype='O')
 
@@ -884,8 +882,10 @@ def map_infer_mask(ndarray arr, object f, ndarray[uint8_t] mask,
         result[i] = val
 
     if convert:
-        return maybe_convert_objects(result, try_float=0,
-                                     convert_datetime=0)
+        return maybe_convert_objects(result,
+                                     try_float=0,
+                                     convert_datetime=0,
+                                     convert_timedelta=0)
 
     return result
 
@@ -920,8 +920,10 @@ def map_infer(ndarray arr, object f, bint convert=1):
         result[i] = val
 
     if convert:
-        return maybe_convert_objects(result, try_float=0,
-                                     convert_datetime=0)
+        return maybe_convert_objects(result,
+                                     try_float=0,
+                                     convert_datetime=0,
+                                     convert_timedelta=0)
 
     return result
 

pandas/tests/test_series.py

@@ -2253,8 +2253,9 @@ def test_constructor_dtype_timedelta64(self):
         td = Series([timedelta(days=1)])
         self.assert_(td.dtype == 'timedelta64[ns]')
 
-        td = Series([timedelta(days=1),timedelta(days=2),np.timedelta64(1,'s')])
-        self.assert_(td.dtype == 'timedelta64[ns]')
+        if not _np_version_under1p7:
+            td = Series([timedelta(days=1),timedelta(days=2),np.timedelta64(1,'s')])
+            self.assert_(td.dtype == 'timedelta64[ns]')
 
         # mixed with NaT
         from pandas import tslib
@@ -2281,8 +2282,9 @@ def test_constructor_dtype_timedelta64(self):
         td = Series([pd.NaT, np.timedelta64(300000000)])
         self.assert_(td.dtype == 'timedelta64[ns]')
 
-        td = Series([np.timedelta64(1,'s')])
-        self.assert_(td.dtype == 'timedelta64[ns]')
+        if not _np_version_under1p7:
+            td = Series([np.timedelta64(1,'s')])
+            self.assert_(td.dtype == 'timedelta64[ns]')
 
         # these are frequency conversion astypes
         #for t in ['s', 'D', 'us', 'ms']:
@@ -2878,6 +2880,9 @@ def test_bfill(self):
         assert_series_equal(ts.bfill(), ts.fillna(method='bfill'))
 
     def test_sub_of_datetime_from_TimeSeries(self):
+        if _np_version_under1p7:
+            raise nose.SkipTest("timedelta broken in np 1.6.1")
+
         from pandas.tseries.timedeltas import _possibly_cast_to_timedelta
         from datetime import datetime
         a = Timestamp(datetime(1993, 0o1, 0o7, 13, 30, 00))

pandas/tools/tests/test_merge.py

@@ -9,7 +9,7 @@
 import random
 
 from pandas.compat import range, lrange, lzip, zip
-from pandas import compat
+from pandas import compat, _np_version_under1p7
 from pandas.tseries.index import DatetimeIndex
 from pandas.tools.merge import merge, concat, ordered_merge, MergeError
 from pandas.util.testing import (assert_frame_equal, assert_series_equal,
@@ -791,8 +791,16 @@ def test_append_dtype_coerce(self):
         result = df1.append(df2,ignore_index=True)
         assert_frame_equal(result, expected)
 
-        # timedelta64
+    def test_join_append_timedeltas(self):
+
+        import datetime as dt
+        from pandas import NaT
+
+        # timedelta64 issues with join/merge
         # GH 5695
+        if _np_version_under1p7:
+            raise nose.SkipTest("numpy < 1.7")
+
         d = {'d': dt.datetime(2013, 11, 5, 5, 56), 't': dt.timedelta(0, 22500)}
         df = DataFrame(columns=list('dt'))
         df = df.append(d, ignore_index=True)
@@ -1787,6 +1795,11 @@ def test_concat_datetime64_block(self):
         self.assert_((result.iloc[10:]['time'] == rng).all())
 
     def test_concat_timedelta64_block(self):
+
+        # not friendly for < 1.7
+        if _np_version_under1p7:
+            raise nose.SkipTest("numpy < 1.7")
+
         from pandas import to_timedelta
 
         rng = to_timedelta(np.arange(10),unit='s')

pandas/tseries/tests/test_timedeltas.py

@@ -174,6 +174,7 @@ def conv(v):
         self.assert_(result == expected)
 
     def test_to_timedelta_via_apply(self):
+        _skip_if_numpy_not_friendly()
 
         # GH 5458
         expected = Series([np.timedelta64(1,'s')])

pandas/tseries/timedeltas.py

@@ -163,7 +163,7 @@ def convert(td, dtype):
                     td *= 1000
                 return td
 
-            if td == tslib.compat_NaT:
+            if isnull(td) or td == tslib.compat_NaT or td == tslib.iNaT:
                 return tslib.iNaT
 
             # convert td value to a nanosecond value

test.py

@@ -0,0 +1,12 @@
+import numpy as np
+import pandas
+from pandas import Series,DataFrame
+
+print pandas.__version__
+
+s = Series(np.arange(1028.))
+
+df = DataFrame({ i:s for i in range(1028) })
+
+import pdb; pdb.set_trace()
+df.apply(lambda x: np.corrcoef(x,s)[0,1])