From f70760856c8a5219fc87ef161b68e73e093a8bef Mon Sep 17 00:00:00 2001 From: Brock Date: Thu, 1 Apr 2021 14:06:04 -0700 Subject: [PATCH 1/2] BUG: DTBlock/TDBlock.delete casting to ndarray --- pandas/core/internals/blocks.py | 22 ++++++++++++++++++++-- pandas/core/internals/concat.py | 6 ++++-- pandas/tests/internals/test_internals.py | 18 ++++++++++++++++++ 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index a77ea61d9e6de..1442fd5f76246 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -280,6 +280,11 @@ def make_block_same_class( """ Wrap given values in a block of same type as self. """ if placement is None: placement = self._mgr_locs + + if values.dtype.kind == "m": + # TODO: remove this once fastparquet has stopped relying on it + values = ensure_wrapped_if_datetimelike(values) + # We assume maybe_coerce_values has already been called return type(self)(values, placement=placement, ndim=self.ndim) @@ -371,7 +376,6 @@ def set_inplace(self, locs, values): """ self.values[locs] = values - @final def delete(self, loc) -> None: """ Delete given loc(-s) from block in-place. @@ -1893,6 +1897,19 @@ def fillna( new_values = values.fillna(value=value, limit=limit) return [self.make_block_same_class(values=new_values)] + def delete(self, loc) -> None: + """ + Delete given loc(-s) from block in-place. + """ + # This will be unnecessary if/when __array_function__ is implemented + self.values = self.values.delete(loc, axis=0) + self.mgr_locs = self._mgr_locs.delete(loc) + try: + self._cache.clear() + except AttributeError: + # _cache not yet initialized + pass + class DatetimeLikeBlockMixin(NDArrayBackedExtensionBlock): """Mixin class for DatetimeBlock, DatetimeTZBlock, and TimedeltaBlock.""" @@ -2173,7 +2190,8 @@ def ensure_block_shape(values: ArrayLike, ndim: int = 1) -> ArrayLike: # TODO(EA2D): https://github.com/pandas-dev/pandas/issues/23023 # block.shape is incorrect for "2D" ExtensionArrays # We can't, and don't need to, reshape. - values = np.asarray(values).reshape(1, -1) + values = values.reshape(1, -1) + return values diff --git a/pandas/core/internals/concat.py b/pandas/core/internals/concat.py index 02d582c439ea2..a5c1f3985e70e 100644 --- a/pandas/core/internals/concat.py +++ b/pandas/core/internals/concat.py @@ -43,6 +43,7 @@ DatetimeArray, ExtensionArray, ) +from pandas.core.construction import ensure_wrapped_if_datetimelike from pandas.core.internals.array_manager import ArrayManager from pandas.core.internals.blocks import ( ensure_block_shape, @@ -142,8 +143,9 @@ def concatenate_managers( else: # TODO(EA2D): special-casing not needed with 2D EAs values = concat_compat(vals) - if not isinstance(values, ExtensionArray): - values = values.reshape(1, len(values)) + values = ensure_block_shape(values, ndim=2) + + values = ensure_wrapped_if_datetimelike(values) if blk.values.dtype == values.dtype: # Fast-path diff --git a/pandas/tests/internals/test_internals.py b/pandas/tests/internals/test_internals.py index c242623520b75..c67ef9177ca96 100644 --- a/pandas/tests/internals/test_internals.py +++ b/pandas/tests/internals/test_internals.py @@ -30,6 +30,7 @@ from pandas.core.arrays import ( DatetimeArray, SparseArray, + TimedeltaArray, ) from pandas.core.internals import ( BlockManager, @@ -300,6 +301,23 @@ def test_delete(self): with pytest.raises(IndexError, match=None): newb.delete(3) + def test_delete_datetimelike(self): + # dont use np.delete on values, as that will coerce from DTA/TDA to ndarray + arr = np.arange(20, dtype="i8").reshape(5, 4).view("m8[ns]") + df = DataFrame(arr) + blk = df._mgr.blocks[0] + assert isinstance(blk.values, TimedeltaArray) + + blk.delete(1) + assert isinstance(blk.values, TimedeltaArray) + + df = DataFrame(arr.view("M8[ns]")) + blk = df._mgr.blocks[0] + assert isinstance(blk.values, DatetimeArray) + + blk.delete([1, 3]) + assert isinstance(blk.values, DatetimeArray) + def test_split(self): # GH#37799 values = np.random.randn(3, 4) From 9444137f54d4cbbc0c32a95dea4c638218c60e34 Mon Sep 17 00:00:00 2001 From: Brock Date: Thu, 1 Apr 2021 16:25:17 -0700 Subject: [PATCH 2/2] mypy fixup --- pandas/core/internals/blocks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index 1442fd5f76246..5ad7e391c2b86 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -2190,6 +2190,7 @@ def ensure_block_shape(values: ArrayLike, ndim: int = 1) -> ArrayLike: # TODO(EA2D): https://github.com/pandas-dev/pandas/issues/23023 # block.shape is incorrect for "2D" ExtensionArrays # We can't, and don't need to, reshape. + values = cast(Union[np.ndarray, DatetimeArray, TimedeltaArray], values) values = values.reshape(1, -1) return values