CLN: cleaned up _possibly_convert_platform

Author: jreback

RELEASE.rst

@@ -53,9 +53,9 @@ pandas 0.11.0
 
   - Do not automatically upcast numeric specified dtypes to ``int64`` or
     ``float64`` (GH622_ and GH797_)
-  - DataFrame construction of lists will no longer be platform dependent when
-    dtype is NOT specified, e.g. DataFrame([1,2]) will be ``int64``
-    like DataFrame({'a' : [1,2]})
+  - DataFrame construction of lists and scalars, with no dtype present, will
+    result in casting to ``int64`` or ``float64``, regardless of platform.
+    This is not an apparent change in the API, but noting it.
   - Guarantee that ``convert_objects()`` for Series/DataFrame always returns a
     copy
   - groupby operations will respect dtypes for numeric float operations

doc/source/v0.11.0.txt

@@ -57,11 +57,17 @@ Numeric dtypes will propagate and can coexist in DataFrames. If a dtype is passe
 
 **Platform Gotchas**
 
-In versions prior to 0.11.0, DataFrame construction with lists was platform dependent (meaning 32-bit vs 64-bit). 
-``DataFrame([1,2],columns=['a'])`` would have a dtype of ``int32``, 
-while ``DataFrame({'a' : [1,2] })`` would be ``int64``. 
-Now construction dtype defaults will be handled in a platform independent manor, 
-resulting in defaults for integers of ``int64`` and floats of ``float64`` dtypes.
+Starting in 0.11.0, construction of DataFrame/Series will use default dtypes of ``int64`` and ``float64``,
+*regardless of platform*. This is not an apparent change from earlier versions of pandas. If you specify
+dtypes, they *WILL* be respected, however.
+
+The following will all result in ``int64`` dtypes
+
+.. ipython:: python
+
+    DataFrame([1,2],columns=['a']).dtypes
+    DataFrame({'a' : [1,2] }.dtypes
+    DataFrame({'a' : 1).dtypes
 
 Keep in mind that ``DataFrame(np.array([1,2]))`` **WILL** result in ``int32`` on 32-bit platforms!
 

pandas/core/common.py

@@ -643,6 +643,21 @@ def take_fast(arr, indexer, mask, needs_masking, axis=0, out=None,
     take_f(arr, indexer, out=out, fill_value=fill_value)
     return out
 
+def _dtype_from_scalar(val):
+    """ interpret the dtype from a scalar, upcast floats and ints """
+    if isinstance(val, np.datetime64):
+        # ugly hacklet
+        val = lib.Timestamp(val).value
+        return val, np.dtype('M8[ns]')
+
+    # provide implicity upcast on scalars
+    elif is_integer(val):
+        if not is_bool(val):
+            return val, np.int64
+    elif is_float(val):
+        return val, np.float64
+
+    return val, type(val)
 
 def _maybe_promote(dtype, fill_value=np.nan):
     if issubclass(dtype.type, np.datetime64):
@@ -654,7 +669,7 @@ def _maybe_promote(dtype, fill_value=np.nan):
         if issubclass(dtype.type, np.bool_):
             return np.object_
         elif issubclass(dtype.type, np.integer):
-            return np.float_
+            return np.float64
         return dtype
     elif is_bool(fill_value):
         if issubclass(dtype.type, np.bool_):
@@ -682,7 +697,7 @@ def _maybe_promote(dtype, fill_value=np.nan):
 def _maybe_upcast(values):
     # TODO: convert remaining usage of _maybe_upcast to _maybe_promote
     if issubclass(values.dtype.type, np.integer):
-        values = values.astype(np.float_)
+        values = values.astype(np.float64)
     elif issubclass(values.dtype.type, np.bool_):
         values = values.astype(np.object_)
     return values
@@ -805,11 +820,11 @@ def _consensus_name_attr(objs):
 # Lots of little utilities
 
 
-def _possibly_convert_objects(values, convert_dates=True, convert_numeric=True, convert_platform=False):
+def _possibly_convert_objects(values, convert_dates=True, convert_numeric=True):
     """ if we have an object dtype, try to coerce dates and/or numers """
 
     # convert dates
-    if convert_dates and getattr(values,'dtype',None) == np.object_:
+    if convert_dates and values.dtype == np.object_:
 
         # we take an aggressive stance and convert to datetime64[ns]
         if convert_dates == 'coerce':
@@ -823,7 +838,7 @@ def _possibly_convert_objects(values, convert_dates=True, convert_numeric=True,
             values = lib.maybe_convert_objects(values, convert_datetime=convert_dates)
 
     # convert to numeric
-    if convert_numeric and getattr(values,'dtype',None) == np.object_:
+    if convert_numeric and values.dtype == np.object_:
         try:
             new_values = lib.maybe_convert_numeric(values,set(),coerce_numeric=True)
 
@@ -834,13 +849,15 @@ def _possibly_convert_objects(values, convert_dates=True, convert_numeric=True,
         except:
             pass
 
-    # platform conversion
-    #   allow ndarray or list here
-    if convert_platform:
-        if isinstance(values, (list,tuple)):
-            values = lib.list_to_object_array(values)
-        if values.dtype == np.object_:
-            values = lib.maybe_convert_objects(values)
+    return values
+
+def _possibly_convert_platform(values):
+    """ try to do platform conversion, allow ndarray or list here """
+
+    if isinstance(values, (list,tuple)):
+        values = lib.list_to_object_array(values)
+    if values.dtype == np.object_:
+        values = lib.maybe_convert_objects(values)
 
     return values
 
@@ -887,12 +904,13 @@ def _possibly_cast_to_datetime(value, dtype, coerce = False):
 
 
 def _infer_dtype(value):
+    # provide upcasting here for floats/ints
     if isinstance(value, (float, np.floating)):
-        return np.float_
+        return np.float64
     elif isinstance(value, (bool, np.bool_)):
         return np.bool_
     elif isinstance(value, (int, long, np.integer)):
-        return np.int_
+        return np.int64
     elif isinstance(value, (complex, np.complexfloating)):
         return np.complex_
     else:

pandas/core/frame.py

@@ -23,13 +23,13 @@
 import numpy.ma as ma
 
 from pandas.core.common import (isnull, notnull, PandasError, _try_sort,
-                                _default_index, _is_sequence)
+                                _default_index, _is_sequence, _dtype_from_scalar)
 from pandas.core.generic import NDFrame
 from pandas.core.index import Index, MultiIndex, _ensure_index
 from pandas.core.indexing import (_NDFrameIndexer, _maybe_droplevels,
                                   _is_index_slice, _check_bool_indexer)
 from pandas.core.internals import BlockManager, make_block, form_blocks
-from pandas.core.series import Series, _radd_compat, _dtype_from_scalar
+from pandas.core.series import Series, _radd_compat
 from pandas.compat.scipy import scoreatpercentile as _quantile
 from pandas.util.compat import OrderedDict
 from pandas.util import py3compat
@@ -2207,21 +2207,29 @@ def _sanitize_column(self, key, value):
             if key in self.columns:
                 existing_piece = self[key]
 
+                # upcast the scalar
+                value, dtype = _dtype_from_scalar(value)
+
                 # transpose hack
                 if isinstance(existing_piece, DataFrame):
                     shape = (len(existing_piece.columns), len(self.index))
                     value = np.repeat(value, np.prod(shape)).reshape(shape)
                 else:
                     value = np.repeat(value, len(self.index))
 
-                    # special case for now
+                    # special case for now (promotion)
                     if (com.is_float_dtype(existing_piece) and
                             com.is_integer_dtype(value)):
-                        value = value.astype(np.float64)
+                        dtype = np.float64
+                        
+                value = value.astype(dtype)
 
             else:
-                value = np.repeat(value, len(self.index))
+                # upcast the scalar
+                value, dtype = _dtype_from_scalar(value)
+                value = np.array(np.repeat(value, len(self.index)), dtype=dtype)
 
+            value = com._possibly_cast_to_datetime(value, dtype)
         return np.atleast_2d(np.asarray(value))
 
     def pop(self, item):
@@ -5461,11 +5469,7 @@ def _prep_ndarray(values, copy=True):
             return np.empty((0, 0), dtype=object)
 
         def convert(v):
-            return com._possibly_convert_objects(v,
-                                                 convert_dates=False,
-                                                 convert_numeric=False,
-                                                 convert_platform=True)
-
+            return com._possibly_convert_platform(v)
 
         # we could have a 1-dim or 2-dim list here
         # this is equiv of np.asarray, but does object conversion

pandas/core/series.py

@@ -3113,10 +3113,7 @@ def _try_cast(arr):
                 subarr = lib.maybe_convert_objects(subarr)
 
         else:
-            subarr = com._possibly_convert_objects(data,
-                                                   convert_dates=False,
-                                                   convert_numeric=False,
-                                                   convert_platform=True)
+            subarr = com._possibly_convert_platform(data)
 
         subarr = com._possibly_cast_to_datetime(subarr, dtype)
 
@@ -3145,7 +3142,7 @@ def _try_cast(arr):
                     dtype = value.dtype
                     value = value.item()
                 else:
-                    value, dtype = _dtype_from_scalar(value)
+                    value, dtype = com._dtype_from_scalar(value)
 
                 subarr = pa.empty(len(index), dtype=dtype)
             else:
@@ -3180,14 +3177,6 @@ def _try_cast(arr):
     return subarr
 
 
-def _dtype_from_scalar(val):
-    if isinstance(val, np.datetime64):
-        # ugly hacklet
-        val = lib.Timestamp(val).value
-        return val, np.dtype('M8[ns]')
-    return val, type(val)
-
-
 def _get_rename_function(mapper):
     if isinstance(mapper, (dict, Series)):
         def f(x):