From 602724d58e125b0b6aaa10412fa3ed04a20e3fd9 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Wed, 1 Apr 2020 20:22:42 -0700 Subject: [PATCH] REF: put concatenate_block_managers in internals.concat --- pandas/core/internals/__init__.py | 2 +- pandas/core/internals/concat.py | 58 ++++++++++++++++++++++++++++--- pandas/core/internals/managers.py | 47 ------------------------- 3 files changed, 54 insertions(+), 53 deletions(-) diff --git a/pandas/core/internals/__init__.py b/pandas/core/internals/__init__.py index e70652b81c42f..3e2f805f003a0 100644 --- a/pandas/core/internals/__init__.py +++ b/pandas/core/internals/__init__.py @@ -14,10 +14,10 @@ _safe_reshape, make_block, ) +from pandas.core.internals.concat import concatenate_block_managers from pandas.core.internals.managers import ( BlockManager, SingleBlockManager, - concatenate_block_managers, create_block_manager_from_arrays, create_block_manager_from_blocks, ) diff --git a/pandas/core/internals/concat.py b/pandas/core/internals/concat.py index 6839d138fbf73..720e6799a3bf3 100644 --- a/pandas/core/internals/concat.py +++ b/pandas/core/internals/concat.py @@ -23,9 +23,57 @@ from pandas.core.dtypes.missing import isna import pandas.core.algorithms as algos +from pandas.core.internals.blocks import make_block +from pandas.core.internals.managers import BlockManager -def get_mgr_concatenation_plan(mgr, indexers): +def concatenate_block_managers( + mgrs_indexers, axes, concat_axis: int, copy: bool +) -> BlockManager: + """ + Concatenate block managers into one. + + Parameters + ---------- + mgrs_indexers : list of (BlockManager, {axis: indexer,...}) tuples + axes : list of Index + concat_axis : int + copy : bool + + Returns + ------- + BlockManager + """ + concat_plans = [ + _get_mgr_concatenation_plan(mgr, indexers) for mgr, indexers in mgrs_indexers + ] + concat_plan = _combine_concat_plans(concat_plans, concat_axis) + blocks = [] + + for placement, join_units in concat_plan: + + if len(join_units) == 1 and not join_units[0].indexers: + b = join_units[0].block + values = b.values + if copy: + values = values.copy() + else: + values = values.view() + b = b.make_block_same_class(values, placement=placement) + elif _is_uniform_join_units(join_units): + b = join_units[0].block.concat_same_type([ju.block for ju in join_units]) + b.mgr_locs = placement + else: + b = make_block( + _concatenate_join_units(join_units, concat_axis, copy=copy), + placement=placement, + ) + blocks.append(b) + + return BlockManager(blocks, axes) + + +def _get_mgr_concatenation_plan(mgr, indexers): """ Construct concatenation plan for given block manager and indexers. @@ -232,7 +280,7 @@ def get_reindexed_values(self, empty_dtype, upcasted_na): return values -def concatenate_join_units(join_units, concat_axis, copy): +def _concatenate_join_units(join_units, concat_axis, copy): """ Concatenate values from several join units along selected axis. """ @@ -371,11 +419,11 @@ def _get_empty_dtype_and_na(join_units): raise AssertionError(msg) -def is_uniform_join_units(join_units) -> bool: +def _is_uniform_join_units(join_units) -> bool: """ Check if the join units consist of blocks of uniform type that can be concatenated using Block.concat_same_type instead of the generic - concatenate_join_units (which uses `concat_compat`). + _concatenate_join_units (which uses `concat_compat`). """ return ( @@ -429,7 +477,7 @@ def _trim_join_unit(join_unit, length): return JoinUnit(block=extra_block, indexers=extra_indexers, shape=extra_shape) -def combine_concat_plans(plans, concat_axis): +def _combine_concat_plans(plans, concat_axis): """ Combine multiple concatenation plans into one. diff --git a/pandas/core/internals/managers.py b/pandas/core/internals/managers.py index 182a5b14a1242..a971d40d2a125 100644 --- a/pandas/core/internals/managers.py +++ b/pandas/core/internals/managers.py @@ -48,12 +48,6 @@ get_block_type, make_block, ) -from pandas.core.internals.concat import ( # all for concatenate_block_managers - combine_concat_plans, - concatenate_join_units, - get_mgr_concatenation_plan, - is_uniform_join_units, -) from pandas.io.formats.printing import pprint_thing @@ -1968,44 +1962,3 @@ def _preprocess_slice_or_indexer(slice_or_indexer, length, allow_fill): if not allow_fill: indexer = maybe_convert_indices(indexer, length) return "fancy", indexer, len(indexer) - - -def concatenate_block_managers(mgrs_indexers, axes, concat_axis, copy): - """ - Concatenate block managers into one. - - Parameters - ---------- - mgrs_indexers : list of (BlockManager, {axis: indexer,...}) tuples - axes : list of Index - concat_axis : int - copy : bool - - """ - concat_plans = [ - get_mgr_concatenation_plan(mgr, indexers) for mgr, indexers in mgrs_indexers - ] - concat_plan = combine_concat_plans(concat_plans, concat_axis) - blocks = [] - - for placement, join_units in concat_plan: - - if len(join_units) == 1 and not join_units[0].indexers: - b = join_units[0].block - values = b.values - if copy: - values = values.copy() - else: - values = values.view() - b = b.make_block_same_class(values, placement=placement) - elif is_uniform_join_units(join_units): - b = join_units[0].block.concat_same_type([ju.block for ju in join_units]) - b.mgr_locs = placement - else: - b = make_block( - concatenate_join_units(join_units, concat_axis, copy=copy), - placement=placement, - ) - blocks.append(b) - - return BlockManager(blocks, axes)