Merge PR #2752

Author: wesm

RELEASE.rst

@@ -44,7 +44,7 @@ pandas 0.10.2
   - Do not automatically upcast numeric specified dtypes to ``int64`` or ``float64`` (GH622_ and GH797_)
   - Guarantee that ``convert_objects()`` for Series/DataFrame always returns a copy
   - groupby operations will respect dtypes for numeric float operations (float32/float64); other types will be operated on,
-    and will try to cast back to the input dtype (e.g. if an int is passed, as long as the output doesn't have nans, 
+    and will try to cast back to the input dtype (e.g. if an int is passed, as long as the output doesn't have nans,
     then an int will be returned)
   - backfill/pad/take/diff/ohlc will now support ``float32/int16/int8`` operations
   - Integer block types will upcast as needed in where operations (GH2793_)
@@ -53,10 +53,23 @@ pandas 0.10.2
 
   - Fix seg fault on empty data frame when fillna with ``pad`` or ``backfill`` (GH2778_)
 
+**API Changes**
+
+  - Series now automatically will try to set the correct dtype based on passed datetimelike objects (datetime/Timestamp)
+     - timedelta64 are returned in appropriate cases (e.g. Series - Series, when both are datetime64)
+     - mixed datetimes and objects (GH2751_) in a constructor witll be casted correctly
+     - astype on datetimes to object are now handled (as well as NaT conversions to np.nan)
+
+**Bug fixes**
+
+  - Single element ndarrays of datetimelike objects are handled (e.g. np.array(datetime(2001,1,1,0,0))), w/o dtype being passed
+  - 0-dim ndarrays with a passed dtype are handled correctly (e.g. np.array(0.,dtype='float32'))
+
 .. _GH622: https://github.com/pydata/pandas/issues/622
 .. _GH797: https://github.com/pydata/pandas/issues/797
 .. _GH2778: https://github.com/pydata/pandas/issues/2778
 .. _GH2793: https://github.com/pydata/pandas/issues/2793
+.. _GH2751: https://github.com/pydata/pandas/issues/2751
 
 pandas 0.10.1
 =============

doc/source/missing_data.rst

@@ -80,6 +80,23 @@ pandas provides the :func:`~pandas.core.common.isnull` and
 missing by the ``isnull`` and ``notnull`` functions. ``inf`` and
 ``-inf`` are no longer considered missing by default.
 
+Datetimes
+---------
+
+For datetime64[ns] types, ``NaT`` represents missing values. This is a pseudo-native
+sentinal value that can be represented by numpy in a singular dtype (datetime64[ns]).
+Pandas objects provide intercompatibility between ``NaT`` and ``NaN``.
+
+.. ipython:: python
+
+   df2 = df.copy()
+   df2['timestamp'] = Timestamp('20120101')
+   df2
+   df2.ix[['a','c','h'],['one','timestamp']] = np.nan
+   df2
+   df2.get_dtype_counts()
+
+
 Calculations with missing data
 ------------------------------
 

doc/source/v0.10.2.txt

@@ -50,7 +50,7 @@ Numeric dtypes will propgate and can coexist in DataFrames. If a dtype is passed
    df3.dtypes
 
    # forcing date coercion
-   s = Series([datetime(2001,1,1,0,0), 'foo', 1.0, 1, 
+   s = Series([datetime(2001,1,1,0,0), 'foo', 1.0, 1,
                Timestamp('20010104'), '20010105'],dtype='O')
    s.convert_objects(convert_dates='coerce')
 
@@ -93,3 +93,33 @@ See the `full release notes
 <https://github.com/pydata/pandas/blob/master/RELEASE.rst>`__ or issue tracker
 on GitHub for a complete list.
 
+
+Datetime64[ns] columns in a DataFrame (or a Series) allow the use of ``np.nan`` to indicate a nan value, in addition to the traditional ``NaT``, or not-a-time. This allows convenient nan setting in a generic way. Furthermore datetime64 columns are created by default, when passed datetimelike objects (*this change was introduced in 0.10.1*)
+
+.. ipython:: python
+
+   df = DataFrame(randn(6,2),date_range('20010102',periods=6),columns=['A','B'])
+   df['timestamp'] = Timestamp('20010103')
+   df
+
+   # datetime64[ns] out of the box
+   df.get_dtype_counts()
+
+   # use the traditional nan, which is mapped to NaT internally
+   df.ix[2:4,['A','timestamp']] = np.nan
+   df
+
+Astype conversion on datetime64[ns] to object, implicity converts ``NaT`` to ``np.nan``
+
+
+.. ipython:: python
+
+   import datetime
+   s = Series([datetime.datetime(2001, 1, 2, 0, 0) for i in range(3)])
+   s.dtype
+   s[1] = np.nan
+   s
+   s.dtype
+   s = s.astype('O')
+   s
+   s.dtype

pandas/core/common.py

@@ -720,6 +720,20 @@ def _possibly_cast_to_datetime(value, dtype, coerce = False):
                 except:
                     pass
 
+    elif dtype is None:
+        # we might have a array (or single object) that is datetime like, and no dtype is passed
+        # don't change the value unless we find a datetime set
+        v = value
+        if not (is_list_like(v) or hasattr(v,'len')):
+            v = [ v ]
+        if len(v):
+            inferred_type = lib.infer_dtype(v)
+            if inferred_type == 'datetime':
+                try:
+                    value = tslib.array_to_datetime(np.array(v))
+                except:
+                    pass
+
     return value
 
 

pandas/core/frame.py

@@ -4300,7 +4300,7 @@ def applymap(self, func):
 
         # if we have a dtype == 'M8[ns]', provide boxed values
         def infer(x):
-            if x.dtype == 'M8[ns]':
+            if com.is_datetime64_dtype(x):
                 x = lib.map_infer(x, lib.Timestamp)
             return lib.map_infer(x, func)
         return self.apply(infer)
@@ -4991,7 +4991,7 @@ def _get_agg_axis(self, axis_num):
     def _get_numeric_data(self):
         if self._is_mixed_type:
             num_data = self._data.get_numeric_data()
-            return DataFrame(num_data, copy=False)
+            return DataFrame(num_data, index=self.index, copy=False)
         else:
             if (self.values.dtype != np.object_ and
                     not issubclass(self.values.dtype.type, np.datetime64)):
@@ -5002,7 +5002,7 @@ def _get_numeric_data(self):
     def _get_bool_data(self):
         if self._is_mixed_type:
             bool_data = self._data.get_bool_data()
-            return DataFrame(bool_data, copy=False)
+            return DataFrame(bool_data, index=self.index, copy=False)
         else:  # pragma: no cover
             if self.values.dtype == np.bool_:
                 return self

pandas/core/series.py

@@ -72,17 +72,28 @@ def na_op(x, y):
 
     def wrapper(self, other):
         from pandas.core.frame import DataFrame
+        dtype = None
         wrap_results = lambda x: x
 
         lvalues, rvalues = self, other
 
-        if (com.is_datetime64_dtype(self) and
-            com.is_datetime64_dtype(other)):
+        if com.is_datetime64_dtype(self):
+
+            if not isinstance(rvalues, np.ndarray):
+                rvalues = np.array([rvalues])
+
+            # rhs is either a timedelta or a series/ndarray
+            if lib.is_timedelta_array(rvalues):
+                rvalues = np.array([ np.timedelta64(v) for v in rvalues ],dtype='timedelta64[ns]')
+                dtype = 'M8[ns]'
+            elif com.is_datetime64_dtype(rvalues):
+                dtype = 'timedelta64[ns]'
+            else:
+                raise ValueError("cannot operate on a series with out a rhs of a series/ndarray of type datetime64[ns] or a timedelta")
+
             lvalues = lvalues.view('i8')
             rvalues = rvalues.view('i8')
 
-            wrap_results = lambda rs: rs.astype('timedelta64[ns]')
-
         if isinstance(rvalues, Series):
             lvalues = lvalues.values
             rvalues = rvalues.values
@@ -91,7 +102,7 @@ def wrapper(self, other):
             if self.index.equals(other.index):
                 name = _maybe_match_name(self, other)
                 return Series(wrap_results(na_op(lvalues, rvalues)),
-                              index=self.index, name=name)
+                              index=self.index, name=name, dtype=dtype)
 
             join_idx, lidx, ridx = self.index.join(other.index, how='outer',
                                                    return_indexers=True)
@@ -105,13 +116,13 @@ def wrapper(self, other):
             arr = na_op(lvalues, rvalues)
 
             name = _maybe_match_name(self, other)
-            return Series(arr, index=join_idx, name=name)
+            return Series(arr, index=join_idx, name=name,dtype=dtype)
         elif isinstance(other, DataFrame):
             return NotImplemented
         else:
             # scalars
             return Series(na_op(lvalues.values, rvalues),
-                          index=self.index, name=self.name)
+                          index=self.index, name=self.name, dtype=dtype)
     return wrapper
 
 
@@ -777,7 +788,7 @@ def astype(self, dtype):
         See numpy.ndarray.astype
         """
         casted = com._astype_nansafe(self.values, dtype)
-        return self._constructor(casted, index=self.index, name=self.name)
+        return self._constructor(casted, index=self.index, name=self.name, dtype=casted.dtype)
 
     def convert_objects(self, convert_dates=True, convert_numeric=True):
         """
@@ -1201,7 +1212,7 @@ def tolist(self):
         Overrides numpy.ndarray.tolist
         """
         if com.is_datetime64_dtype(self):
-            return self.astype(object).values.tolist()
+            return list(self)
         return self.values.tolist()
 
     def to_dict(self):
@@ -3103,8 +3114,12 @@ def _try_cast(arr):
                         raise TypeError('Cannot cast datetime64 to %s' % dtype)
                 else:
                     subarr = _try_cast(data)
-        elif copy:
+        else:
+            subarr = _try_cast(data)
+
+        if copy:
             subarr = data.copy()
+
     elif isinstance(data, list) and len(data) > 0:
         if dtype is not None:
             try:
@@ -3114,12 +3129,15 @@ def _try_cast(arr):
                     raise
                 subarr = np.array(data, dtype=object, copy=copy)
                 subarr = lib.maybe_convert_objects(subarr)
+                subarr = com._possibly_cast_to_datetime(subarr, dtype)
         else:
             subarr = lib.list_to_object_array(data)
             subarr = lib.maybe_convert_objects(subarr)
+            subarr = com._possibly_cast_to_datetime(subarr, dtype)
     else:
         subarr = _try_cast(data)
 
+    # scalar like
     if subarr.ndim == 0:
         if isinstance(data, list):  # pragma: no cover
             subarr = np.array(data, dtype=object)
@@ -3135,7 +3153,14 @@ def _try_cast(arr):
                 dtype = np.object_
 
             if dtype is None:
-                value, dtype = _dtype_from_scalar(value)
+                
+                # a 1-element ndarray
+                if isinstance(value, np.ndarray):
+                    dtype = value.dtype
+                    value = value.item()
+                else:
+                    value, dtype = _dtype_from_scalar(value)
+
                 subarr = np.empty(len(index), dtype=dtype)
             else:
                 # need to possibly convert the value here
@@ -3144,6 +3169,17 @@ def _try_cast(arr):
             subarr.fill(value)
         else:
             return subarr.item()
+
+    # the result that we want
+    elif subarr.ndim == 1:
+        if index is not None:
+
+            # a 1-element ndarray
+            if len(subarr) != len(index) and len(subarr) == 1:
+                value = subarr[0]
+                subarr = np.empty(len(index), dtype=subarr.dtype)
+                subarr.fill(value)
+
     elif subarr.ndim > 1:
         if isinstance(data, np.ndarray):
             raise Exception('Data must be 1-dimensional')

pandas/src/inference.pyx

@@ -265,6 +265,17 @@ def is_datetime64_array(ndarray values):
             return False
     return True
 
+def is_timedelta_array(ndarray values):
+    import datetime
+    cdef int i, n = len(values)
+    if n == 0:
+        return False
+    for i in range(n):
+        if not isinstance(values[i],datetime.timedelta):
+            return False
+    return True
+
+
 def is_date_array(ndarray[object] values):
     cdef int i, n = len(values)
     if n == 0: