Skip to content

REF: put concatenate_block_managers in internals.concat #33231

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Apr 6, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 2 additions & 5 deletions pandas/core/internals/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,8 @@
_safe_reshape,
make_block,
)
from pandas.core.internals.managers import (
BlockManager,
SingleBlockManager,
concatenate_block_managers,
)
from pandas.core.internals.concat import concatenate_block_managers
from pandas.core.internals.managers import BlockManager, SingleBlockManager

__all__ = [
"Block",
Expand Down
58 changes: 53 additions & 5 deletions pandas/core/internals/concat.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,57 @@
from pandas.core.dtypes.missing import isna

import pandas.core.algorithms as algos
from pandas.core.internals.blocks import make_block
from pandas.core.internals.managers import BlockManager


def get_mgr_concatenation_plan(mgr, indexers):
def concatenate_block_managers(
mgrs_indexers, axes, concat_axis: int, copy: bool
) -> BlockManager:
"""
Concatenate block managers into one.

Parameters
----------
mgrs_indexers : list of (BlockManager, {axis: indexer,...}) tuples
axes : list of Index
concat_axis : int
copy : bool

Returns
-------
BlockManager
"""
concat_plans = [
_get_mgr_concatenation_plan(mgr, indexers) for mgr, indexers in mgrs_indexers
]
concat_plan = _combine_concat_plans(concat_plans, concat_axis)
blocks = []

for placement, join_units in concat_plan:

if len(join_units) == 1 and not join_units[0].indexers:
b = join_units[0].block
values = b.values
if copy:
values = values.copy()
else:
values = values.view()
b = b.make_block_same_class(values, placement=placement)
elif _is_uniform_join_units(join_units):
b = join_units[0].block.concat_same_type([ju.block for ju in join_units])
b.mgr_locs = placement
else:
b = make_block(
_concatenate_join_units(join_units, concat_axis, copy=copy),
placement=placement,
)
blocks.append(b)

return BlockManager(blocks, axes)


def _get_mgr_concatenation_plan(mgr, indexers):
"""
Construct concatenation plan for given block manager and indexers.

Expand Down Expand Up @@ -232,7 +280,7 @@ def get_reindexed_values(self, empty_dtype, upcasted_na):
return values


def concatenate_join_units(join_units, concat_axis, copy):
def _concatenate_join_units(join_units, concat_axis, copy):
"""
Concatenate values from several join units along selected axis.
"""
Expand Down Expand Up @@ -371,11 +419,11 @@ def _get_empty_dtype_and_na(join_units):
raise AssertionError(msg)


def is_uniform_join_units(join_units) -> bool:
def _is_uniform_join_units(join_units) -> bool:
"""
Check if the join units consist of blocks of uniform type that can
be concatenated using Block.concat_same_type instead of the generic
concatenate_join_units (which uses `concat_compat`).
_concatenate_join_units (which uses `concat_compat`).

"""
return (
Expand Down Expand Up @@ -429,7 +477,7 @@ def _trim_join_unit(join_unit, length):
return JoinUnit(block=extra_block, indexers=extra_indexers, shape=extra_shape)


def combine_concat_plans(plans, concat_axis):
def _combine_concat_plans(plans, concat_axis):
"""
Combine multiple concatenation plans into one.

Expand Down
47 changes: 0 additions & 47 deletions pandas/core/internals/managers.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,12 +47,6 @@
get_block_type,
make_block,
)
from pandas.core.internals.concat import ( # all for concatenate_block_managers
combine_concat_plans,
concatenate_join_units,
get_mgr_concatenation_plan,
is_uniform_join_units,
)

from pandas.io.formats.printing import pprint_thing

Expand Down Expand Up @@ -2002,44 +1996,3 @@ def _preprocess_slice_or_indexer(slice_or_indexer, length, allow_fill):
if not allow_fill:
indexer = maybe_convert_indices(indexer, length)
return "fancy", indexer, len(indexer)


def concatenate_block_managers(mgrs_indexers, axes, concat_axis, copy):
"""
Concatenate block managers into one.

Parameters
----------
mgrs_indexers : list of (BlockManager, {axis: indexer,...}) tuples
axes : list of Index
concat_axis : int
copy : bool

"""
concat_plans = [
get_mgr_concatenation_plan(mgr, indexers) for mgr, indexers in mgrs_indexers
]
concat_plan = combine_concat_plans(concat_plans, concat_axis)
blocks = []

for placement, join_units in concat_plan:

if len(join_units) == 1 and not join_units[0].indexers:
b = join_units[0].block
values = b.values
if copy:
values = values.copy()
else:
values = values.view()
b = b.make_block_same_class(values, placement=placement)
elif is_uniform_join_units(join_units):
b = join_units[0].block.concat_same_type([ju.block for ju in join_units])
b.mgr_locs = placement
else:
b = make_block(
concatenate_join_units(join_units, concat_axis, copy=copy),
placement=placement,
)
blocks.append(b)

return BlockManager(blocks, axes)