API/PERF: add policy argument to constructors, pandas-dev#10556

- closes pandas-dev#10556, add policy argument to constructors
- closes pandas-dev#9216, all passing of dict with view directly to the API
- closes pandas-dev#5902

Author: jreback

pandas/core/config_init.py

@@ -407,16 +407,26 @@ def use_inf_as_na_cb(key):
                     rkey='mode.use_inf_as_na')
 
 
-# user warnings
+#
+# options from the "mode" namespace
+
 chained_assignment = """
 : string
     Raise an exception, warn, or no action if trying to use chained assignment,
     The default is warn
 """
 
+policy = """
+: string
+    Default policy for construction of objects,
+    The default is 'block'
+"""
+
 with cf.config_prefix('mode'):
     cf.register_option('chained_assignment', 'warn', chained_assignment,
                        validator=is_one_of_factory([None, 'warn', 'raise']))
+    cf.register_option('policy', 'block', policy,
+                       validator=is_one_of_factory(['block', 'column', 'split']))
 
 # Set up the io.excel specific configuration.
 writer_engine_doc = """

pandas/core/frame.py

@@ -266,6 +266,12 @@ class DataFrame(NDFrame):
         Data type to force. Only a single dtype is allowed. If None, infer
     copy : boolean, default False
         Copy data from inputs. Only affects DataFrame / 2d ndarray input
+    policy : string, default None
+        Provide consolidation policy
+          - None : use default policy
+          - block : consolidate into blocks by dtype
+          - column : don't consolidate, but don't split blocks
+          - split : don't consolidate, force splitting of input
 
     Examples
     --------
@@ -327,7 +333,7 @@ def _constructor_expanddim(self):
         return Panel
 
     def __init__(self, data=None, index=None, columns=None, dtype=None,
-                 copy=False):
+                 copy=False, policy=None):
         if data is None:
             data = {}
         if dtype is not None:
@@ -338,9 +344,10 @@ def __init__(self, data=None, index=None, columns=None, dtype=None,
 
         if isinstance(data, BlockManager):
             mgr = self._init_mgr(data, axes=dict(index=index, columns=columns),
-                                 dtype=dtype, copy=copy)
+                                 dtype=dtype, copy=copy, policy=policy)
         elif isinstance(data, dict):
-            mgr = self._init_dict(data, index, columns, dtype=dtype)
+            mgr = self._init_dict(data, index, columns, dtype=dtype,
+                                  policy=policy)
         elif isinstance(data, ma.MaskedArray):
             import numpy.ma.mrecords as mrecords
             # masked recarray
@@ -357,7 +364,7 @@ def __init__(self, data=None, index=None, columns=None, dtype=None,
                 else:
                     data = data.copy()
                 mgr = self._init_ndarray(data, index, columns, dtype=dtype,
-                                         copy=copy)
+                                         copy=copy, policy=policy)
 
         elif isinstance(data, (np.ndarray, Series, Index)):
             if data.dtype.names:
@@ -368,10 +375,10 @@ def __init__(self, data=None, index=None, columns=None, dtype=None,
                 mgr = self._init_dict(data, index, columns, dtype=dtype)
             elif getattr(data, 'name', None) is not None:
                 mgr = self._init_dict({data.name: data}, index, columns,
-                                      dtype=dtype)
+                                      dtype=dtype, policy=policy)
             else:
                 mgr = self._init_ndarray(data, index, columns, dtype=dtype,
-                                         copy=copy)
+                                         copy=copy, policy=policy)
         elif isinstance(data, (list, types.GeneratorType)):
             if isinstance(data, types.GeneratorType):
                 data = list(data)
@@ -392,12 +399,13 @@ def __init__(self, data=None, index=None, columns=None, dtype=None,
                             index = _default_index(len(data))
 
                     mgr = _arrays_to_mgr(arrays, columns, index, columns,
-                                         dtype=dtype)
+                                         dtype=dtype, policy=policy)
                 else:
                     mgr = self._init_ndarray(data, index, columns, dtype=dtype,
-                                             copy=copy)
+                                             copy=copy, policy=policy)
             else:
-                mgr = self._init_dict({}, index, columns, dtype=dtype)
+                mgr = self._init_dict({}, index, columns, dtype=dtype,
+                                      policy=policy)
         elif isinstance(data, collections.Iterator):
             raise TypeError("data argument can't be an iterator")
         else:
@@ -412,13 +420,14 @@ def __init__(self, data=None, index=None, columns=None, dtype=None,
                 values = cast_scalar_to_array((len(index), len(columns)),
                                               data, dtype=dtype)
                 mgr = self._init_ndarray(values, index, columns,
-                                         dtype=values.dtype, copy=False)
+                                         dtype=values.dtype, copy=False,
+                                         policy=policy)
             else:
                 raise ValueError('DataFrame constructor not properly called!')
 
         NDFrame.__init__(self, mgr, fastpath=True)
 
-    def _init_dict(self, data, index, columns, dtype=None):
+    def _init_dict(self, data, index, columns, dtype=None, policy=None):
         """
         Segregate Series based on type and coerce into matrices.
         Needs to handle a lot of exceptional cases.
@@ -470,9 +479,11 @@ def _init_dict(self, data, index, columns, dtype=None):
             columns = data_names = Index(keys)
             arrays = [data[k] for k in keys]
 
-        return _arrays_to_mgr(arrays, data_names, index, columns, dtype=dtype)
+        return _arrays_to_mgr(arrays, data_names, index, columns, dtype=dtype,
+                              policy=policy)
 
-    def _init_ndarray(self, values, index, columns, dtype=None, copy=False):
+    def _init_ndarray(self, values, index, columns, dtype=None, copy=False,
+                      policy=None):
         # input must be a ndarray, list, Series, index
 
         if isinstance(values, Series):
@@ -542,7 +553,8 @@ def _get_axes(N, K, index=index, columns=columns):
         if dtype is None and is_object_dtype(values):
             values = maybe_infer_to_datetimelike(values)
 
-        return create_block_manager_from_blocks([values], [columns, index])
+        return create_block_manager_from_blocks([values], [columns, index],
+                                                policy=policy)
 
     @property
     def axes(self):
@@ -6018,7 +6030,7 @@ def isin(self, values):
 ops.add_special_arithmetic_methods(DataFrame, **ops.frame_special_funcs)
 
 
-def _arrays_to_mgr(arrays, arr_names, index, columns, dtype=None):
+def _arrays_to_mgr(arrays, arr_names, index, columns, dtype=None, policy=None):
     """
     Segregate Series based on type and coerce into matrices.
     Needs to handle a lot of exceptional cases.
@@ -6035,7 +6047,8 @@ def _arrays_to_mgr(arrays, arr_names, index, columns, dtype=None):
     # from BlockManager perspective
     axes = [_ensure_index(columns), _ensure_index(index)]
 
-    return create_block_manager_from_arrays(arrays, arr_names, axes)
+    return create_block_manager_from_arrays(arrays, arr_names, axes,
+                                            policy=policy)
 
 
 def extract_index(data):

pandas/core/generic.py

@@ -159,6 +159,19 @@ def _repr_data_resource_(self):
                                  object_pairs_hook=collections.OrderedDict)
             return payload
 
+    @property
+    def _policy(self):
+        """ return my policy for internal implementation """
+        return self._data.policy
+
+    @_policy.setter
+    def _policy(self, value):
+        """
+        set my policy for internal implementation
+        should only set the property for state purposes
+        """
+        self._data.policy = value
+
     def _validate_dtype(self, dtype):
         """ validate the passed dtype """
 
@@ -173,7 +186,7 @@ def _validate_dtype(self, dtype):
 
         return dtype
 
-    def _init_mgr(self, mgr, axes=None, dtype=None, copy=False):
+    def _init_mgr(self, mgr, axes=None, dtype=None, copy=False, policy=None):
         """ passed a manager and a axes dict """
         for a, axe in axes.items():
             if axe is not None:
@@ -1515,7 +1528,7 @@ def to_dense(self):
     def __getstate__(self):
         meta = {k: getattr(self, k, None) for k in self._metadata}
         return dict(_data=self._data, _typ=self._typ, _metadata=self._metadata,
-                    **meta)
+                    _policy=self._policy, **meta)
 
     def __setstate__(self, state):
 
@@ -2310,9 +2323,17 @@ def _check_is_chained_assignment_possible(self):
         """
         if self._is_view and self._is_cached:
             ref = self._get_cacher()
-            if ref is not None and ref._is_mixed_type:
-                self._check_setitem_copy(stacklevel=4, t='referant',
-                                         force=True)
+            if ref is not None:
+
+                # TODO: fix me!
+                # if we are a single block, then we don't need to check
+                # anything here if we are column and are actually a block,
+                # maybe be a bit tricky
+                if ref._policy in ['column', 'split']:
+                    return True
+                if ref._is_mixed_type:
+                    self._check_setitem_copy(stacklevel=4, t='referant',
+                                             force=True)
             return True
         elif self._is_copy:
             self._check_setitem_copy(stacklevel=4, t='referant')
@@ -4436,7 +4457,7 @@ def copy(self, deep=True):
         copy : type of caller
         """
         data = self._data.copy(deep=deep)
-        return self._constructor(data).__finalize__(self)
+        return self._constructor(data, policy=self._policy).__finalize__(self)
 
     def __copy__(self, deep=True):
         return self.copy(deep=deep)