API/PERF: add policy argument to constructors, pandas-dev#10556

- closes pandas-dev#10556, add policy argument to constructors
- closes pandas-dev#9216, all passing of dict with view directly to the API
- closes pandas-dev#5902
- closes pandas-dev#8571 by defining __copy__/__deepcopy__

Author: jreback

pandas/core/config_init.py

@@ -333,16 +333,26 @@ def use_inf_as_null_cb(key):
                        cb=use_inf_as_null_cb)
 
 
-# user warnings
+#
+# options from the "mode" namespace
+
 chained_assignment = """
 : string
     Raise an exception, warn, or no action if trying to use chained assignment,
     The default is warn
 """
 
+policy = """
+: string
+    Default policy for construction of objects,
+    The default is 'block'
+"""
+
 with cf.config_prefix('mode'):
     cf.register_option('chained_assignment', 'warn', chained_assignment,
                        validator=is_one_of_factory([None, 'warn', 'raise']))
+    cf.register_option('policy', 'block', policy,
+                       validator=is_one_of_factory(['block', 'column', 'split']))
 
 
 # Set up the io.excel specific configuration.

pandas/core/frame.py

@@ -182,6 +182,12 @@ class DataFrame(NDFrame):
         Data type to force, otherwise infer
     copy : boolean, default False
         Copy data from inputs. Only affects DataFrame / 2d ndarray input
+    policy : string, default None
+        Provide consolidation policy
+          - None : use default policy
+          - block : consolidate into blocks by dtype
+          - column : don't consolidate, but don't split blocks
+          - split : don't consolidate, force splitting of input
 
     Examples
     --------
@@ -211,7 +217,7 @@ def _constructor_expanddim(self):
         return Panel
 
     def __init__(self, data=None, index=None, columns=None, dtype=None,
-                 copy=False):
+                 copy=False, policy=None):
         if data is None:
             data = {}
         if dtype is not None:
@@ -222,9 +228,9 @@ def __init__(self, data=None, index=None, columns=None, dtype=None,
 
         if isinstance(data, BlockManager):
             mgr = self._init_mgr(data, axes=dict(index=index, columns=columns),
-                                 dtype=dtype, copy=copy)
+                                 dtype=dtype, copy=copy, policy=policy)
         elif isinstance(data, dict):
-            mgr = self._init_dict(data, index, columns, dtype=dtype)
+            mgr = self._init_dict(data, index, columns, dtype=dtype, policy=policy)
         elif isinstance(data, ma.MaskedArray):
             import numpy.ma.mrecords as mrecords
             # masked recarray
@@ -241,7 +247,7 @@ def __init__(self, data=None, index=None, columns=None, dtype=None,
                 else:
                     data = data.copy()
                 mgr = self._init_ndarray(data, index, columns, dtype=dtype,
-                                         copy=copy)
+                                         copy=copy, policy=policy)
 
         elif isinstance(data, (np.ndarray, Series, Index)):
             if data.dtype.names:
@@ -252,10 +258,10 @@ def __init__(self, data=None, index=None, columns=None, dtype=None,
                 mgr = self._init_dict(data, index, columns, dtype=dtype)
             elif getattr(data, 'name', None):
                 mgr = self._init_dict({data.name: data}, index, columns,
-                                      dtype=dtype)
+                                      dtype=dtype, policy=policy)
             else:
                 mgr = self._init_ndarray(data, index, columns, dtype=dtype,
-                                         copy=copy)
+                                         copy=copy, policy=policy)
         elif isinstance(data, (list, types.GeneratorType)):
             if isinstance(data, types.GeneratorType):
                 data = list(data)
@@ -274,12 +280,12 @@ def __init__(self, data=None, index=None, columns=None, dtype=None,
                             index = _default_index(len(data))
 
                     mgr = _arrays_to_mgr(arrays, columns, index, columns,
-                                         dtype=dtype)
+                                         dtype=dtype, policy=policy)
                 else:
                     mgr = self._init_ndarray(data, index, columns, dtype=dtype,
-                                             copy=copy)
+                                             copy=copy, policy=policy)
             else:
-                mgr = self._init_dict({}, index, columns, dtype=dtype)
+                mgr = self._init_dict({}, index, columns, dtype=dtype, policy=policy)
         elif isinstance(data, collections.Iterator):
             raise TypeError("data argument can't be an iterator")
         else:
@@ -299,13 +305,13 @@ def __init__(self, data=None, index=None, columns=None, dtype=None,
                 values = np.empty((len(index), len(columns)), dtype=dtype)
                 values.fill(data)
                 mgr = self._init_ndarray(values, index, columns, dtype=dtype,
-                                         copy=False)
+                                         copy=False, policy=policy)
             else:
                 raise PandasError('DataFrame constructor not properly called!')
 
         NDFrame.__init__(self, mgr, fastpath=True)
 
-    def _init_dict(self, data, index, columns, dtype=None):
+    def _init_dict(self, data, index, columns, dtype=None, policy=None):
         """
         Segregate Series based on type and coerce into matrices.
         Needs to handle a lot of exceptional cases.
@@ -359,10 +365,10 @@ def _init_dict(self, data, index, columns, dtype=None):
             arrays = [data[k] for k in keys]
 
         return _arrays_to_mgr(arrays, data_names, index, columns,
-                              dtype=dtype)
+                              dtype=dtype, policy=policy)
 
     def _init_ndarray(self, values, index, columns, dtype=None,
-                      copy=False):
+                      copy=False, policy=None):
         # input must be a ndarray, list, Series, index
 
         if isinstance(values, Series):
@@ -433,7 +439,7 @@ def _get_axes(N, K, index=index, columns=columns):
         if dtype is None and is_object_dtype(values):
             values = _possibly_infer_to_datetimelike(values)
 
-        return create_block_manager_from_blocks([values], [columns, index])
+        return create_block_manager_from_blocks([values], [columns, index], policy=policy)
 
     @property
     def axes(self):
@@ -5082,7 +5088,7 @@ def combineMult(self, other):
 
 _EMPTY_SERIES = Series([])
 
-def _arrays_to_mgr(arrays, arr_names, index, columns, dtype=None):
+def _arrays_to_mgr(arrays, arr_names, index, columns, dtype=None, policy=None):
     """
     Segregate Series based on type and coerce into matrices.
     Needs to handle a lot of exceptional cases.
@@ -5099,7 +5105,7 @@ def _arrays_to_mgr(arrays, arr_names, index, columns, dtype=None):
     # from BlockManager perspective
     axes = [_ensure_index(columns), _ensure_index(index)]
 
-    return create_block_manager_from_arrays(arrays, arr_names, axes)
+    return create_block_manager_from_arrays(arrays, arr_names, axes, policy=policy)
 
 
 def extract_index(data):

pandas/core/generic.py

@@ -106,6 +106,19 @@ def __init__(self, data, axes=None, copy=False, dtype=None,
         object.__setattr__(self, '_data', data)
         object.__setattr__(self, '_item_cache', {})
 
+    @property
+    def _policy(self):
+        """ return my policy for internal implementation """
+        return self._data.policy
+
+    @_policy.setter
+    def _policy(self, value):
+        """
+        set my policy for internal implementation
+        should only set the property for state purposes
+        """
+        self._data.policy = value
+
     def _validate_dtype(self, dtype):
         """ validate the passed dtype """
 
@@ -119,7 +132,7 @@ def _validate_dtype(self, dtype):
                                           .format(self.__class__.__name__))
         return dtype
 
-    def _init_mgr(self, mgr, axes=None, dtype=None, copy=False):
+    def _init_mgr(self, mgr, axes=None, dtype=None, copy=False, policy=None):
         """ passed a manager and a axes dict """
         for a, axe in axes.items():
             if axe is not None:
@@ -778,7 +791,8 @@ def to_dense(self):
     def __getstate__(self):
         meta = dict((k, getattr(self, k, None)) for k in self._metadata)
         return dict(_data=self._data, _typ=self._typ,
-                    _metadata=self._metadata, **meta)
+                    _metadata=self._metadata,
+                    _policy=self._policy, **meta)
 
     def __setstate__(self, state):
 
@@ -1228,8 +1242,15 @@ def _check_is_chained_assignment_possible(self):
         """
         if self._is_view and self._is_cached:
             ref = self._get_cacher()
-            if ref is not None and ref._is_mixed_type:
-                self._check_setitem_copy(stacklevel=4, t='referant', force=True)
+            if ref is not None:
+
+                # TODO: fix me!
+                # if we are a single block, then we don't need to check anything here
+                # if we are column and are actually a block, maybe be a bit tricky
+                if ref._policy in ['column','split']:
+                    return True
+                if ref._is_mixed_type:
+                    self._check_setitem_copy(stacklevel=4, t='referant', force=True)
             return True
         elif self.is_copy:
             self._check_setitem_copy(stacklevel=4, t='referant')
@@ -2518,6 +2539,12 @@ def astype(self, dtype, copy=True, raise_on_error=True, **kwargs):
             dtype=dtype, copy=copy, raise_on_error=raise_on_error, **kwargs)
         return self._constructor(mgr).__finalize__(self)
 
+    def __deepcopy__(self, memo={}):
+        return self.copy(deep=True)
+
+    def __copy__(self):
+        return self.copy()
+
     def copy(self, deep=True):
         """
         Make a copy of this object
@@ -2532,7 +2559,7 @@ def copy(self, deep=True):
         copy : type of caller
         """
         data = self._data.copy(deep=deep)
-        return self._constructor(data).__finalize__(self)
+        return self._constructor(data, policy=self._policy).__finalize__(self)
 
     def _convert(self, datetime=False, numeric=False, timedelta=False,
                  coerce=False, copy=True):