From 62bbf211c6a37adb29345a8d6fc0d9fd9397fdcd Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sun, 14 Jan 2018 18:38:15 -0800 Subject: [PATCH 1/9] remove unused itemsize; de-duplicate DatetimeTZBlock methods --- pandas/core/internals.py | 34 +++++++++------------------------- 1 file changed, 9 insertions(+), 25 deletions(-) diff --git a/pandas/core/internals.py b/pandas/core/internals.py index 3c923133477df..ea49ee07e6828 100644 --- a/pandas/core/internals.py +++ b/pandas/core/internals.py @@ -303,10 +303,6 @@ def getitem_block(self, slicer, new_mgr_locs=None): def shape(self): return self.values.shape - @property - def itemsize(self): - return self.values.itemsize - @property def dtype(self): return self.values.dtype @@ -1929,6 +1925,7 @@ def should_store(self, value): class DatetimeLikeBlockMixin(object): + freq = None # compat with Datetimelike Index subclasses @property def _na_value(self): @@ -2466,6 +2463,7 @@ class DatetimeBlock(DatetimeLikeBlockMixin, Block): __slots__ = () is_datetime = True _can_hold_na = True + tz = None def __init__(self, values, placement, fastpath=False, **kwargs): if values.dtype != _NS_DTYPE: @@ -2559,7 +2557,7 @@ def _try_coerce_result(self, result): @property def _box_func(self): - return tslib.Timestamp + return lambda x: tslib.Timestamp(x, freq=self.freq, tz=self.tz) def to_native_types(self, slicer=None, na_rep=None, date_format=None, quoting=None, **kwargs): @@ -2603,6 +2601,12 @@ class DatetimeTZBlock(NonConsolidatableMixIn, DatetimeBlock): _concatenator = staticmethod(_concat._concat_datetime) is_datetimetz = True + get_values = DatetimeBlock.get_values # override NonConsolidatableMixin + + @property + def tz(self): + return self.dtype.tz + def __init__(self, values, placement, ndim=2, **kwargs): if not isinstance(values, self._holder): @@ -2634,14 +2638,6 @@ def external_values(self): """ return self.values.astype('datetime64[ns]').values - def get_values(self, dtype=None): - # return object dtype as Timestamps with the zones - if is_object_dtype(dtype): - f = lambda x: lib.Timestamp(x, tz=self.values.tz) - return lib.map_infer( - self.values.ravel(), f).reshape(self.values.shape) - return self.values - def _slice(self, slicer): """ return a slice of my values """ if isinstance(slicer, tuple): @@ -2713,10 +2709,6 @@ def _try_coerce_result(self, result): return result - @property - def _box_func(self): - return lambda x: tslib.Timestamp(x, tz=self.dtype.tz) - def shift(self, periods, axis=0, mgr=None): """ shift the block by periods """ @@ -2767,10 +2759,6 @@ class SparseBlock(NonConsolidatableMixIn, Block): def shape(self): return (len(self.mgr_locs), self.sp_index.length) - @property - def itemsize(self): - return self.dtype.itemsize - @property def fill_value(self): # return np.nan @@ -4542,10 +4530,6 @@ def asobject(self): """ return self._block.get_values(dtype=object) - @property - def itemsize(self): - return self._block.values.itemsize - @property def _can_hold_na(self): return self._block._can_hold_na From 495b522ee8bc3e9dc1dad5577ed4436ecf15ee25 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Mon, 15 Jan 2018 10:42:42 -0800 Subject: [PATCH 2/9] remove unused reindex_axis methods --- pandas/core/internals.py | 43 ++++++---------------------------------- 1 file changed, 6 insertions(+), 37 deletions(-) diff --git a/pandas/core/internals.py b/pandas/core/internals.py index ea49ee07e6828..8dcfec63561e2 100644 --- a/pandas/core/internals.py +++ b/pandas/core/internals.py @@ -323,21 +323,6 @@ def concat_same_type(self, to_concat, placement=None): return self.make_block_same_class( values, placement=placement or slice(0, len(values), 1)) - def reindex_axis(self, indexer, method=None, axis=1, fill_value=None, - limit=None, mask_info=None): - """ - Reindex using pre-computed indexer information - """ - if axis < 1: - raise AssertionError( - 'axis must be at least 1, got {axis}'.format(axis=axis)) - if fill_value is None: - fill_value = self.fill_value - - new_values = algos.take_nd(self.values, indexer, axis, - fill_value=fill_value, mask_info=mask_info) - return self.make_block(new_values, fastpath=True) - def iget(self, i): return self.values[i] @@ -933,10 +918,10 @@ def putmask(self, mask, new, align=True, inplace=False, axis=0, new_values = self.values if inplace else self.values.copy() - if hasattr(new, 'reindex_axis'): + if hasattr(new, 'values'): new = new.values - if hasattr(mask, 'reindex_axis'): + if hasattr(mask, 'values'): mask = mask.values # if we are passed a scalar None, convert it here @@ -1295,7 +1280,7 @@ def eval(self, func, other, errors='raise', try_cast=False, mgr=None): orig_other = other values = self.values - if hasattr(other, 'reindex_axis'): + if hasattr(other, 'values'): other = other.values # make sure that we can broadcast @@ -1444,10 +1429,10 @@ def where(self, other, cond, align=True, errors='raise', if transpose: values = values.T - if hasattr(other, 'reindex_axis'): + if hasattr(other, 'values'): other = other.values - if hasattr(cond, 'reindex_axis'): + if hasattr(cond, 'values'): cond = cond.values # If the default broadcasting would go in the wrong direction, then @@ -2883,22 +2868,6 @@ def shift(self, periods, axis=0, mgr=None): return [self.make_block_same_class(new_values, placement=self.mgr_locs)] - def reindex_axis(self, indexer, method=None, axis=1, fill_value=None, - limit=None, mask_info=None): - """ - Reindex using pre-computed indexer information - """ - if axis < 1: - raise AssertionError( - 'axis must be at least 1, got {axis}'.format(axis=axis)) - - # taking on the 0th axis always here - if fill_value is None: - fill_value = self.fill_value - return self.make_block_same_class(self.values.take(indexer), - fill_value=fill_value, - placement=self.mgr_locs) - def sparse_reindex(self, new_index): """ sparse reindex and return a new block current reindex only works for float64 dtype! """ @@ -3299,7 +3268,7 @@ def apply(self, f, axes=None, filter=None, do_integrity_check=False, aligned_args = dict((k, kwargs[k]) for k in align_keys - if hasattr(kwargs[k], 'reindex_axis')) + if hasattr(kwargs[k], 'values')) for b in self.blocks: if filter is not None: From b229159cae0c806edc358a4dfc19d5bf0867eec6 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Mon, 15 Jan 2018 17:41:06 -0800 Subject: [PATCH 3/9] requested edits --- pandas/core/internals.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/pandas/core/internals.py b/pandas/core/internals.py index 8dcfec63561e2..bb59f202dbba1 100644 --- a/pandas/core/internals.py +++ b/pandas/core/internals.py @@ -918,11 +918,8 @@ def putmask(self, mask, new, align=True, inplace=False, axis=0, new_values = self.values if inplace else self.values.copy() - if hasattr(new, 'values'): - new = new.values - - if hasattr(mask, 'values'): - mask = mask.values + new = getattr(new, 'values', new) + mask = getattr(mask, 'values', mask) # if we are passed a scalar None, convert it here if not is_list_like(new) and isna(new) and not self.is_object: @@ -1280,8 +1277,7 @@ def eval(self, func, other, errors='raise', try_cast=False, mgr=None): orig_other = other values = self.values - if hasattr(other, 'values'): - other = other.values + other = getattr(other, 'values', other) # make sure that we can broadcast is_transposed = False @@ -1429,11 +1425,8 @@ def where(self, other, cond, align=True, errors='raise', if transpose: values = values.T - if hasattr(other, 'values'): - other = other.values - - if hasattr(cond, 'values'): - cond = cond.values + other = getattr(other, 'values', other) + cond = getattr(cond, 'values', cond) # If the default broadcasting would go in the wrong direction, then # explicitly reshape other instead @@ -1910,7 +1903,9 @@ def should_store(self, value): class DatetimeLikeBlockMixin(object): - freq = None # compat with Datetimelike Index subclasses + # We add a dummy `freq` attribute here to make these block subclasses + # behave more like the DatetimelikeIndexOpsMixin implementations + freq = None @property def _na_value(self): @@ -2586,8 +2581,6 @@ class DatetimeTZBlock(NonConsolidatableMixIn, DatetimeBlock): _concatenator = staticmethod(_concat._concat_datetime) is_datetimetz = True - get_values = DatetimeBlock.get_values # override NonConsolidatableMixin - @property def tz(self): return self.dtype.tz @@ -2610,6 +2603,13 @@ def __init__(self, values, placement, ndim=2, **kwargs): super(DatetimeTZBlock, self).__init__(values, placement=placement, ndim=ndim, **kwargs) + def get_values(self, dtype=None): + """ + return object dtype as boxed values, such as Timestamps/Timedelta + """ + # Note: this overrides the NonConsolidatableMixIn implementation + return DatetimeBlock.get_values(self, dtype) + def copy(self, deep=True, mgr=None): """ copy constructor """ values = self.values From fb8c9ea9ba0d92e9d7b30fa4251c70fb261f6264 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Mon, 15 Jan 2018 17:42:20 -0800 Subject: [PATCH 4/9] extra comment --- pandas/core/internals.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pandas/core/internals.py b/pandas/core/internals.py index bb59f202dbba1..ebfeb23cfc3e9 100644 --- a/pandas/core/internals.py +++ b/pandas/core/internals.py @@ -2443,6 +2443,9 @@ class DatetimeBlock(DatetimeLikeBlockMixin, Block): __slots__ = () is_datetime = True _can_hold_na = True + + # We add a dummy `tz` attribute here to make these block subclasses + # behave more like the DatetimelikeIndexOpsMixin implementations tz = None def __init__(self, values, placement, fastpath=False, **kwargs): From 6c4df2ebd08c661c478b2a04f59b581bb3656ce9 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 16 Jan 2018 10:18:40 -0800 Subject: [PATCH 5/9] remove freq --- pandas/core/internals.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pandas/core/internals.py b/pandas/core/internals.py index ebfeb23cfc3e9..ca4aae37c78e5 100644 --- a/pandas/core/internals.py +++ b/pandas/core/internals.py @@ -1903,10 +1903,6 @@ def should_store(self, value): class DatetimeLikeBlockMixin(object): - # We add a dummy `freq` attribute here to make these block subclasses - # behave more like the DatetimelikeIndexOpsMixin implementations - freq = None - @property def _na_value(self): return tslib.NaT @@ -2540,7 +2536,7 @@ def _try_coerce_result(self, result): @property def _box_func(self): - return lambda x: tslib.Timestamp(x, freq=self.freq, tz=self.tz) + return lambda x: tslib.Timestamp(x, tz=self.tz) def to_native_types(self, slicer=None, na_rep=None, date_format=None, quoting=None, **kwargs): From 97a60dd20b8d209e743f8181960630b4e19d9e69 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Wed, 17 Jan 2018 17:08:40 -0800 Subject: [PATCH 6/9] revert box_func de-duplication per request --- pandas/core/internals.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pandas/core/internals.py b/pandas/core/internals.py index ca4aae37c78e5..6f3345cc2ecc8 100644 --- a/pandas/core/internals.py +++ b/pandas/core/internals.py @@ -2440,10 +2440,6 @@ class DatetimeBlock(DatetimeLikeBlockMixin, Block): is_datetime = True _can_hold_na = True - # We add a dummy `tz` attribute here to make these block subclasses - # behave more like the DatetimelikeIndexOpsMixin implementations - tz = None - def __init__(self, values, placement, fastpath=False, **kwargs): if values.dtype != _NS_DTYPE: values = conversion.ensure_datetime64ns(values) @@ -2536,7 +2532,7 @@ def _try_coerce_result(self, result): @property def _box_func(self): - return lambda x: tslib.Timestamp(x, tz=self.tz) + return tslib.Timestamp def to_native_types(self, slicer=None, na_rep=None, date_format=None, quoting=None, **kwargs): @@ -2693,6 +2689,10 @@ def _try_coerce_result(self, result): return result + @property + def _box_func(self): + return lambda x: tslib.Timestamp(x, tz=self.dtype.tz) + def shift(self, periods, axis=0, mgr=None): """ shift the block by periods """ From 2256b5d8364eb700f9c79d2713de8a1068ccda8f Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 18 Jan 2018 08:49:02 -0800 Subject: [PATCH 7/9] revert duplicated get_values method --- pandas/core/internals.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/pandas/core/internals.py b/pandas/core/internals.py index 6f3345cc2ecc8..f3b0a8f7b2408 100644 --- a/pandas/core/internals.py +++ b/pandas/core/internals.py @@ -2598,12 +2598,8 @@ def __init__(self, values, placement, ndim=2, **kwargs): super(DatetimeTZBlock, self).__init__(values, placement=placement, ndim=ndim, **kwargs) - def get_values(self, dtype=None): - """ - return object dtype as boxed values, such as Timestamps/Timedelta - """ - # Note: this overrides the NonConsolidatableMixIn implementation - return DatetimeBlock.get_values(self, dtype) + # override NonConsolidatableMixIn implementation of get_values + get_values = DatetimeLikeBlockMixin.get_values def copy(self, deep=True, mgr=None): """ copy constructor """ From d8f11048190e534cb0b85aee61293e87383fb479 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sun, 21 Jan 2018 09:58:39 -0800 Subject: [PATCH 8/9] revert to use redundant get_values method --- pandas/core/internals.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pandas/core/internals.py b/pandas/core/internals.py index 35778082f11e7..4f18fa07a39d8 100644 --- a/pandas/core/internals.py +++ b/pandas/core/internals.py @@ -2587,9 +2587,6 @@ def __init__(self, values, placement, ndim=2, dtype=None): super(DatetimeTZBlock, self).__init__(values, placement=placement, ndim=ndim) - # override NonConsolidatableMixIn implementation of get_values - get_values = DatetimeLikeBlockMixin.get_values - def copy(self, deep=True, mgr=None): """ copy constructor """ values = self.values @@ -2603,6 +2600,13 @@ def external_values(self): """ return self.values.astype('datetime64[ns]').values + def get_values(self, dtype=None): + # return object dtype as Timestamps with the zones + if is_object_dtype(dtype): + return lib.map_infer( + self.values.ravel(), self._box_func).reshape(self.values.shape) + return self.values + def _slice(self, slicer): """ return a slice of my values """ if isinstance(slicer, tuple): From 59b3e604dd7983d961e02454b1aefd17dd992228 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sun, 21 Jan 2018 09:59:32 -0800 Subject: [PATCH 9/9] revert whitespace change --- pandas/core/internals.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pandas/core/internals.py b/pandas/core/internals.py index 4f18fa07a39d8..5c3481ed6d4ff 100644 --- a/pandas/core/internals.py +++ b/pandas/core/internals.py @@ -1900,6 +1900,7 @@ def should_store(self, value): class DatetimeLikeBlockMixin(object): + @property def _na_value(self): return tslib.NaT