API/PERF: add policy argument to constructors, pandas-dev#10556

- closes pandas-dev#10556, add policy argument to constructors
- closes pandas-dev#9216, all passing of dict with view directly to the API
- closes pandas-dev#5902

Author: jreback

pandas/core/config_init.py

@@ -472,7 +472,9 @@ def use_inf_as_na_cb(key):
 )
 
 
-# user warnings
+#
+# options from the "mode" namespace
+
 chained_assignment = """
 : string
     Raise an exception, warn, or no action if trying to use chained assignment,
@@ -500,6 +502,17 @@ def use_inf_as_na_cb(key):
 _xlsx_options = ["xlrd", "openpyxl"]
 _ods_options = ["odf"]
 _xlsb_options = ["pyxlsb"]
+policy = """
+: string
+    Default policy for construction of objects,
+    The default is 'block'
+"""
+
+with cf.config_prefix('mode'):
+    cf.register_option('chained_assignment', 'warn', chained_assignment,
+                       validator=is_one_of_factory([None, 'warn', 'raise']))
+    cf.register_option('policy', 'block', policy,
+                       validator=is_one_of_factory(['block', 'column', 'split']))
 
 
 with cf.config_prefix("io.excel.xls"):

pandas/core/frame.py

@@ -355,6 +355,12 @@ class DataFrame(NDFrame):
         Data type to force. Only a single dtype is allowed. If None, infer.
     copy : bool, default False
         Copy data from inputs. Only affects DataFrame / 2d ndarray input.
+    policy : string, default None
+        Provide consolidation policy
+          - None : use default policy
+          - block : consolidate into blocks by dtype
+          - column : don't consolidate, but don't split blocks
+          - split : don't consolidate, force splitting of input
 
     See Also
     --------
@@ -363,6 +369,9 @@ class DataFrame(NDFrame):
     read_csv : Read a comma-separated values (csv) file into DataFrame.
     read_table : Read general delimited file into DataFrame.
     read_clipboard : Read text from clipboard into DataFrame.
+        Data type to force. Only a single dtype is allowed. If None, infer
+    copy : boolean, default False
+        Copy data from inputs. Only affects DataFrame / 2d ndarray input
 
     Examples
     --------
@@ -426,6 +435,7 @@ def __init__(
         columns: Optional[Axes] = None,
         dtype: Optional[Dtype] = None,
         copy: bool = False,
+        policy=None,
     ):
         if data is None:
             data = {}
@@ -437,10 +447,11 @@ def __init__(
 
         if isinstance(data, BlockManager):
             mgr = self._init_mgr(
-                data, axes=dict(index=index, columns=columns), dtype=dtype, copy=copy
+                data, axes=dict(index=index, columns=columns), dtype=dtype, copy=copy,
+                policy=policy,
             )
         elif isinstance(data, dict):
-            mgr = init_dict(data, index, columns, dtype=dtype)
+            mgr = init_dict(data, index, columns, dtype=dtype, policy=policy)
         elif isinstance(data, ma.MaskedArray):
             import numpy.ma.mrecords as mrecords
 
@@ -457,19 +468,19 @@ def __init__(
                     data[mask] = fill_value
                 else:
                     data = data.copy()
-                mgr = init_ndarray(data, index, columns, dtype=dtype, copy=copy)
+                mgr = init_ndarray(data, index, columns, dtype=dtype, copy=copy, policy=policy)
 
         elif isinstance(data, (np.ndarray, Series, Index)):
             if data.dtype.names:
                 data_columns = list(data.dtype.names)
                 data = {k: data[k] for k in data_columns}
                 if columns is None:
                     columns = data_columns
-                mgr = init_dict(data, index, columns, dtype=dtype)
+                mgr = init_dict(data, index, columns, dtype=dtype, policy=policy)
             elif getattr(data, "name", None) is not None:
-                mgr = init_dict({data.name: data}, index, columns, dtype=dtype)
+                mgr = init_dict({data.name: data}, index, columns, dtype=dtype, policy=policy)
             else:
-                mgr = init_ndarray(data, index, columns, dtype=dtype, copy=copy)
+                mgr = init_ndarray(data, index, columns, dtype=dtype, copy=copy, policy=policy)
 
         # For data is list-like, or Iterable (will consume into list)
         elif isinstance(data, abc.Iterable) and not isinstance(data, (str, bytes)):
@@ -493,11 +504,11 @@ def __init__(
                         else:
                             index = ibase.default_index(len(data))
 
-                    mgr = arrays_to_mgr(arrays, columns, index, columns, dtype=dtype)
+                    mgr = arrays_to_mgr(arrays, columns, index, columns, dtype=dtype, policy=policy)
                 else:
-                    mgr = init_ndarray(data, index, columns, dtype=dtype, copy=copy)
+                    mgr = init_ndarray(data, index, columns, dtype=dtype, copy=copy, policy=policy)
             else:
-                mgr = init_dict({}, index, columns, dtype=dtype)
+                mgr = init_dict({}, index, columns, dtype=dtype, policy=policy)
         else:
             try:
                 arr = np.array(data, dtype=dtype, copy=copy)
@@ -513,7 +524,7 @@ def __init__(
                     (len(index), len(columns)), data, dtype=dtype
                 )
                 mgr = init_ndarray(
-                    values, index, columns, dtype=values.dtype, copy=False
+                    values, index, columns, dtype=values.dtype, copy=False, policy=policy,
                 )
             else:
                 raise ValueError("DataFrame constructor not properly called!")
@@ -575,7 +586,7 @@ def _is_homogeneous_type(self) -> bool:
         Index._is_homogeneous_type : Whether the object has a single
             dtype.
         MultiIndex._is_homogeneous_type : Whether all the levels of a
-            MultiIndex have the same dtype.
+             have the same dtype.
 
         Examples
         --------

pandas/core/generic.py

@@ -248,6 +248,21 @@ def attrs(self) -> Dict[Optional[Hashable], Any]:
     @attrs.setter
     def attrs(self, value: Mapping[Optional[Hashable], Any]) -> None:
         self._attrs = dict(value)
+    @property
+    def _policy(self):
+        """ return my policy for internal implementation """
+        return self._data.policy
+
+    @_policy.setter
+    def _policy(self, value):
+        """
+        set my policy for internal implementation
+        should only set the property for state purposes
+        """
+        self._data.policy = value
+
+    def _validate_dtype(self, dtype):
+        """ validate the passed dtype """
 
     @classmethod
     def _validate_dtype(cls, dtype):
@@ -1822,6 +1837,7 @@ def __getstate__(self) -> Dict[str, Any]:
             _typ=self._typ,
             _metadata=self._metadata,
             attrs=self.attrs,
+            _policy=self._policy,
             **meta,
         )
 
@@ -3259,8 +3275,133 @@ def _maybe_update_cacher(
         if clear:
             self._clear_item_cache()
 
+<<<<<<< HEAD
     def _clear_item_cache(self) -> None:
         self._item_cache.clear()
+=======
+    def _clear_item_cache(self, i=None):
+        if i is not None:
+            self._item_cache.pop(i, None)
+        else:
+            self._item_cache.clear()
+
+    def _slice(self, slobj, axis=0, kind=None):
+        """
+        Construct a slice of this container.
+
+        kind parameter is maintained for compatibility with Series slicing.
+        """
+        axis = self._get_block_manager_axis(axis)
+        result = self._constructor(self._data.get_slice(slobj, axis=axis))
+        result = result.__finalize__(self)
+
+        # this could be a view
+        # but only in a single-dtyped view slicable case
+        is_copy = axis != 0 or result._is_view
+        result._set_is_copy(self, copy=is_copy)
+        return result
+
+    def _set_item(self, key, value):
+        self._data.set(key, value)
+        self._clear_item_cache()
+
+    def _set_is_copy(self, ref=None, copy=True):
+        if not copy:
+            self._is_copy = None
+        else:
+            if ref is not None:
+                self._is_copy = weakref.ref(ref)
+            else:
+                self._is_copy = None
+
+    def _check_is_chained_assignment_possible(self):
+        """
+        Check if we are a view, have a cacher, and are of mixed type.
+        If so, then force a setitem_copy check.
+
+        Should be called just near setting a value
+
+        Will return a boolean if it we are a view and are cached, but a
+        single-dtype meaning that the cacher should be updated following
+        setting.
+        """
+        if self._is_view and self._is_cached:
+            ref = self._get_cacher()
+            if ref is not None:
+
+                # TODO: fix me!
+                # if we are a single block, then we don't need to check
+                # anything here if we are column and are actually a block,
+                # maybe be a bit tricky
+                if ref._policy in ['column', 'split']:
+                    return True
+                if ref._is_mixed_type:
+                    self._check_setitem_copy(stacklevel=4, t='referant',
+                                             force=True)
+            return True
+        elif self._is_copy:
+            self._check_setitem_copy(stacklevel=4, t='referant')
+        return False
+
+    def _check_setitem_copy(self, stacklevel=4, t='setting', force=False):
+        """
+
+        Parameters
+        ----------
+        stacklevel : integer, default 4
+           the level to show of the stack when the error is output
+        t : string, the type of setting error
+        force : boolean, default False
+           if True, then force showing an error
+
+        validate if we are doing a settitem on a chained copy.
+
+        If you call this function, be sure to set the stacklevel such that the
+        user will see the error *at the level of setting*
+
+        It is technically possible to figure out that we are setting on
+        a copy even WITH a multi-dtyped pandas object. In other words, some
+        blocks may be views while other are not. Currently _is_view will ALWAYS
+        return False for multi-blocks to avoid having to handle this case.
+
+        df = DataFrame(np.arange(0,9), columns=['count'])
+        df['group'] = 'b'
+
+        # This technically need not raise SettingWithCopy if both are view
+        # (which is not # generally guaranteed but is usually True.  However,
+        # this is in general not a good practice and we recommend using .loc.
+        df.iloc[0:5]['group'] = 'a'
+
+        """
+
+        if force or self._is_copy:
+
+            value = config.get_option('mode.chained_assignment')
+            if value is None:
+                return
+
+            # see if the copy is not actually referred; if so, then dissolve
+            # the copy weakref
+            try:
+                gc.collect(2)
+                if not gc.get_referents(self._is_copy()):
+                    self._is_copy = None
+                    return
+            except Exception:
+                pass
+
+            # we might be a false positive
+            try:
+                if self._is_copy().shape == self.shape:
+                    self._is_copy = None
+                    return
+            except Exception:
+                pass
+
+            # a custom message
+            if isinstance(self._is_copy, string_types):
+                t = self._is_copy
+>>>>>>> API/PERF: add policy argument to constructors, #10556
 
     # ----------------------------------------------------------------------
     # Indexing Methods
@@ -5682,7 +5823,7 @@ def copy(self: FrameOrSeries, deep: bool_t = True) -> FrameOrSeries:
         dtype: object
         """
         data = self._data.copy(deep=deep)
-        return self._constructor(data).__finalize__(self)
+        return self._constructor(data, policy=self._policy).__finalize__(self)
 
     def __copy__(self: FrameOrSeries, deep: bool_t = True) -> FrameOrSeries:
         return self.copy(deep=deep)