diff --git a/pandas/core/indexing.py b/pandas/core/indexing.py index 9df72053fb0af..c61ef2e3d1e60 100644 --- a/pandas/core/indexing.py +++ b/pandas/core/indexing.py @@ -6,34 +6,41 @@ import pandas.core.common as com from pandas.core.common import (is_bool_indexer, is_integer_dtype, _asarray_tuplesafe, is_list_like, isnull, - is_null_slice, is_full_slice, - ABCSeries, ABCDataFrame, ABCPanel, is_float, - _values_from_object, _infer_fill_value, is_integer) + is_null_slice, is_full_slice, ABCSeries, + ABCDataFrame, ABCPanel, is_float, + _values_from_object, _infer_fill_value, + is_integer) import numpy as np + # the supported indexers def get_indexers_list(): return [ - ('ix', _IXIndexer), + ('ix', _IXIndexer), ('iloc', _iLocIndexer), - ('loc', _LocIndexer), - ('at', _AtIndexer), - ('iat', _iAtIndexer), + ('loc', _LocIndexer), + ('at', _AtIndexer), + ('iat', _iAtIndexer), ] # "null slice" _NS = slice(None, None) + # the public IndexSlicerMaker class _IndexSlice(object): def __getitem__(self, arg): return arg + + IndexSlice = _IndexSlice() + class IndexingError(Exception): pass + class _NDFrameIndexer(object): _valid_types = None _exception = KeyError @@ -50,7 +57,7 @@ def __call__(self, *args, **kwargs): # set the passed in values for k, v in compat.iteritems(kwargs): - setattr(self,k,v) + setattr(self, k, v) return self def __iter__(self): @@ -79,8 +86,7 @@ def _get_label(self, label, axis=0): return self.obj._xs(label, axis=axis) except: return self.obj[label] - elif (isinstance(label, tuple) and - isinstance(label[axis], slice)): + elif isinstance(label, tuple) and isinstance(label[axis], slice): raise IndexingError('no slices here, handle elsewhere') return self.obj._xs(label, axis=axis) @@ -129,7 +135,9 @@ def _has_valid_tuple(self, key): "types" % self._valid_types) def _should_validate_iterable(self, axis=0): - """ return a boolean whether this axes needs validation for a passed iterable """ + """ return a boolean whether this axes needs validation for a passed + iterable + """ ax = self.obj._get_axis(axis) if isinstance(ax, MultiIndex): return False @@ -139,8 +147,8 @@ def _should_validate_iterable(self, axis=0): return True def _is_nested_tuple_indexer(self, tup): - if any([ isinstance(ax, MultiIndex) for ax in self.obj.axes ]): - return any([ is_nested_tuple(tup,ax) for ax in self.obj.axes ]) + if any([isinstance(ax, MultiIndex) for ax in self.obj.axes]): + return any([is_nested_tuple(tup, ax) for ax in self.obj.axes]) return False def _convert_tuple(self, key, is_setter=False): @@ -149,7 +157,8 @@ def _convert_tuple(self, key, is_setter=False): axis = self.obj._get_axis_number(self.axis) for i in range(self.ndim): if i == axis: - keyidx.append(self._convert_to_indexer(key, axis=axis, is_setter=is_setter)) + keyidx.append(self._convert_to_indexer( + key, axis=axis, is_setter=is_setter)) else: keyidx.append(slice(None)) else: @@ -178,7 +187,8 @@ def _has_valid_setitem_indexer(self, indexer): def _has_valid_positional_setitem_indexer(self, indexer): """ validate that an positional indexer cannot enlarge its target - will raise if needed, does not modify the indexer externally """ + will raise if needed, does not modify the indexer externally + """ if isinstance(indexer, dict): raise IndexError("{0} cannot enlarge its target object" .format(self.name)) @@ -206,7 +216,8 @@ def _setitem_with_indexer(self, indexer, value): self._has_valid_setitem_indexer(indexer) # also has the side effect of consolidating in-place - from pandas import Panel, DataFrame, Series + # TODO: Panel, DataFrame are not imported, remove? + from pandas import Panel, DataFrame, Series # noqa info_axis = self.obj._info_axis_number # maybe partial set @@ -217,16 +228,19 @@ def _setitem_with_indexer(self, indexer, value): if not take_split_path and self.obj._data.blocks: blk, = self.obj._data.blocks if 1 < blk.ndim: # in case of dict, keys are indices - val = list(value.values()) if isinstance(value,dict) else value + val = list(value.values()) if isinstance(value, + dict) else value take_split_path = not blk._can_hold_element(val) if isinstance(indexer, tuple) and len(indexer) == len(self.obj.axes): for i, ax in zip(indexer, self.obj.axes): - # if we have any multi-indexes that have non-trivial slices (not null slices) - # then we must take the split path, xref GH 10360 - if isinstance(ax, MultiIndex) and not (is_integer(i) or is_null_slice(i)): + # if we have any multi-indexes that have non-trivial slices + # (not null slices) then we must take the split path, xref + # GH 10360 + if (isinstance(ax, MultiIndex) and + not (is_integer(i) or is_null_slice(i))): take_split_path = True break @@ -261,7 +275,6 @@ def _setitem_with_indexer(self, indexer, value): self.obj[key] = value return self.obj - # add a new item with the dtype setup self.obj[key] = _infer_fill_value(value) @@ -276,10 +289,10 @@ def _setitem_with_indexer(self, indexer, value): # just replacing the block manager here # so the object is the same index = self.obj._get_axis(i) - labels = index.insert(len(index),key) + labels = index.insert(len(index), key) self.obj._data = self.obj.reindex_axis(labels, i)._data self.obj._maybe_update_cacher(clear=True) - self.obj.is_copy=None + self.obj.is_copy = None nindexer.append(labels.get_loc(key)) @@ -297,7 +310,7 @@ def _setitem_with_indexer(self, indexer, value): # and set inplace if self.ndim == 1: index = self.obj.index - new_index = index.insert(len(index),indexer) + new_index = index.insert(len(index), indexer) # this preserves dtype of the value new_values = Series([value])._values @@ -314,14 +327,14 @@ def _setitem_with_indexer(self, indexer, value): # no columns and scalar if not len(self.obj.columns): - raise ValueError( - "cannot set a frame with no defined columns" - ) + raise ValueError("cannot set a frame with no defined " + "columns") # append a Series if isinstance(value, Series): - value = value.reindex(index=self.obj.columns,copy=True) + value = value.reindex(index=self.obj.columns, + copy=True) value.name = indexer # a list-list @@ -330,11 +343,11 @@ def _setitem_with_indexer(self, indexer, value): # must have conforming columns if is_list_like_indexer(value): if len(value) != len(self.obj.columns): - raise ValueError( - "cannot set a row with mismatched columns" - ) + raise ValueError("cannot set a row with " + "mismatched columns") - value = Series(value,index=self.obj.columns,name=indexer) + value = Series(value, index=self.obj.columns, + name=indexer) self.obj._data = self.obj.append(value)._data self.obj._maybe_update_cacher(clear=True) @@ -375,23 +388,24 @@ def _setitem_with_indexer(self, indexer, value): # require that we are setting the right number of values that # we are indexing - if is_list_like_indexer(value) and np.iterable(value) and lplane_indexer != len(value): + if is_list_like_indexer(value) and np.iterable( + value) and lplane_indexer != len(value): if len(obj[idx]) != len(value): - raise ValueError( - "cannot set using a multi-index selection indexer " - "with a different length than the value" - ) + raise ValueError("cannot set using a multi-index " + "selection indexer with a different " + "length than the value") # make sure we have an ndarray - value = getattr(value,'values',value).ravel() + value = getattr(value, 'values', value).ravel() # we can directly set the series here # as we select a slice indexer on the mi idx = index._convert_slice_indexer(idx) obj._consolidate_inplace() obj = obj.copy() - obj._data = obj._data.setitem(indexer=tuple([idx]), value=value) + obj._data = obj._data.setitem(indexer=tuple([idx]), + value=value) self.obj[item] = obj return @@ -411,9 +425,13 @@ def setter(item, v): # perform the equivalent of a setitem on the info axis # as we have a null slice or a slice with full bounds - # which means essentially reassign to the columns of a multi-dim object + # which means essentially reassign to the columns of a + # multi-dim object # GH6149 (null slice), GH10408 (full bounds) - if isinstance(pi, tuple) and all(is_null_slice(idx) or is_full_slice(idx, len(self.obj)) for idx in pi): + if (isinstance(pi, tuple) and + all(is_null_slice(idx) or + is_full_slice(idx, len(self.obj)) + for idx in pi)): s = v else: # set the item, possibly having a dtype change @@ -444,7 +462,7 @@ def can_do_equal_len(): # we need an iterable, with a ndim of at least 1 # eg. don't pass through np.array(0) - if is_list_like_indexer(value) and getattr(value,'ndim',1) > 0: + if is_list_like_indexer(value) and getattr(value, 'ndim', 1) > 0: # we have an equal len Frame if isinstance(value, ABCDataFrame) and value.ndim > 1: @@ -455,8 +473,8 @@ def can_do_equal_len(): if item in value: sub_indexer[info_axis] = item v = self._align_series( - tuple(sub_indexer), value[item], multiindex_indexer - ) + tuple(sub_indexer), value[item], + multiindex_indexer) else: v = np.nan @@ -467,7 +485,7 @@ def can_do_equal_len(): # note that this coerces the dtype if we are mixed # GH 7551 - value = np.array(value,dtype=object) + value = np.array(value, dtype=object) if len(labels) != value.shape[1]: raise ValueError('Must have equal len keys and value ' 'when setting with an ndarray') @@ -503,8 +521,10 @@ def can_do_equal_len(): # if we are setting on the info axis ONLY # set using those methods to avoid block-splitting # logic here - if len(indexer) > info_axis and is_integer(indexer[info_axis]) and all( - is_null_slice(idx) for i, idx in enumerate(indexer) if i != info_axis): + if (len(indexer) > info_axis and + is_integer(indexer[info_axis]) and + all(is_null_slice(idx) for i, idx in enumerate(indexer) + if i != info_axis)): self.obj[item_labels[indexer[info_axis]]] = value return @@ -522,7 +542,8 @@ def can_do_equal_len(): # actually do the set self.obj._consolidate_inplace() - self.obj._data = self.obj._data.setitem(indexer=indexer, value=value) + self.obj._data = self.obj._data.setitem(indexer=indexer, + value=value) self.obj._maybe_update_cacher(clear=True) def _align_series(self, indexer, ser, multiindex_indexer=False): @@ -658,7 +679,8 @@ def _align_frame(self, indexer, df): aligners = [not is_null_slice(idx) for idx in indexer] sum_aligners = sum(aligners) - single_aligner = sum_aligners == 1 + # TODO: single_aligner is not used + single_aligner = sum_aligners == 1 # noqa idx, cols = None, None sindexers = [] @@ -697,8 +719,8 @@ def _align_frame(self, indexer, df): val = df.reindex(idx, columns=cols)._values return val - elif ((isinstance(indexer, slice) or is_list_like_indexer(indexer)) - and is_frame): + elif ((isinstance(indexer, slice) or is_list_like_indexer(indexer)) and + is_frame): ax = self.obj.index[indexer] if df.index.equals(ax): val = df.copy()._values @@ -706,9 +728,11 @@ def _align_frame(self, indexer, df): # we have a multi-index and are trying to align # with a particular, level GH3738 - if isinstance(ax, MultiIndex) and isinstance( - df.index, MultiIndex) and ax.nlevels != df.index.nlevels: - raise TypeError("cannot align on a multi-index with out specifying the join levels") + if (isinstance(ax, MultiIndex) and + isinstance(df.index, MultiIndex) and + ax.nlevels != df.index.nlevels): + raise TypeError("cannot align on a multi-index with out " + "specifying the join levels") val = df.reindex(index=ax)._values return val @@ -728,8 +752,9 @@ def _align_frame(self, indexer, df): raise ValueError('Incompatible indexer with DataFrame') def _align_panel(self, indexer, df): - is_frame = self.obj.ndim == 2 - is_panel = self.obj.ndim >= 3 + # TODO: is_frame, is_panel are unused + is_frame = self.obj.ndim == 2 # noqa + is_panel = self.obj.ndim >= 3 # noqa raise NotImplementedError("cannot set using an indexer with a Panel " "yet!") @@ -786,10 +811,9 @@ def _multi_take(self, tup): """ try: o = self.obj - d = dict([ - (a, self._convert_for_reindex(t, axis=o._get_axis_number(a))) - for t, a in zip(tup, o._AXIS_ORDERS) - ]) + d = dict( + [(a, self._convert_for_reindex(t, axis=o._get_axis_number(a))) + for t, a in zip(tup, o._AXIS_ORDERS)]) return o.reindex(**d) except: raise self._exception @@ -876,8 +900,8 @@ def _getitem_lowerdim(self, tup): # unfortunately need an odious kludge here because of # DataFrame transposing convention - if (isinstance(section, ABCDataFrame) and i > 0 - and len(new_key) == 2): + if (isinstance(section, ABCDataFrame) and i > 0 and + len(new_key) == 2): a, b = new_key new_key = b, a @@ -901,7 +925,8 @@ def _getitem_nested_tuple(self, tup): if result is not None: return result - # this is a series with a multi-index specified a tuple of selectors + # this is a series with a multi-index specified a tuple of + # selectors return self._getitem_axis(tup, axis=0) # handle the multi-axis by taking sections and reducing @@ -919,7 +944,7 @@ def _getitem_nested_tuple(self, tup): axis += 1 # if we have a scalar, we are done - if np.isscalar(obj) or not hasattr(obj,'ndim'): + if np.isscalar(obj) or not hasattr(obj, 'ndim'): break # has the dim of the obj changed? @@ -944,8 +969,9 @@ def _getitem_axis(self, key, axis=0): labels = self.obj._get_axis(axis) if isinstance(key, slice): return self._get_slice_axis(key, axis=axis) - elif is_list_like_indexer(key) and not (isinstance(key, tuple) and - isinstance(labels, MultiIndex)): + elif (is_list_like_indexer(key) and + not (isinstance(key, tuple) and + isinstance(labels, MultiIndex))): if hasattr(key, 'ndim') and key.ndim > 1: raise ValueError('Cannot index with multidimensional key') @@ -1004,13 +1030,16 @@ def _getitem_iterable(self, key, axis=0): if labels.is_unique and Index(keyarr).is_unique: try: - result = self.obj.reindex_axis(keyarr, axis=axis, level=level) + result = self.obj.reindex_axis(keyarr, axis=axis, + level=level) # this is an error as we are trying to find # keys in a multi-index that don't exist if isinstance(labels, MultiIndex) and level is not None: - if hasattr(result,'ndim') and not np.prod(result.shape) and len(keyarr): - raise KeyError("cannot index a multi-index axis with these keys") + if (hasattr(result, 'ndim') and + not np.prod(result.shape) and len(keyarr)): + raise KeyError("cannot index a multi-index axis " + "with these keys") return result @@ -1029,19 +1058,19 @@ def _getitem_iterable(self, key, axis=0): raise AssertionError("invalid indexing error with " "non-unique index") - new_target, indexer, new_indexer = labels._reindex_non_unique(keyarr) + new_target, indexer, new_indexer = labels._reindex_non_unique( + keyarr) if new_indexer is not None: - result = self.obj.take(indexer[indexer!=-1], axis=axis, + result = self.obj.take(indexer[indexer != -1], axis=axis, convert=False) - result = result._reindex_with_indexers({ - axis: [new_target, new_indexer] - }, copy=True, allow_dups=True) + result = result._reindex_with_indexers( + {axis: [new_target, new_indexer]}, + copy=True, allow_dups=True) else: - result = self.obj.take(indexer, axis=axis, - convert=False) + result = self.obj.take(indexer, axis=axis, convert=False) return result @@ -1182,7 +1211,6 @@ def _get_slice_axis(self, slice_obj, axis=0): class _IXIndexer(_NDFrameIndexer): - """A primarily label-location based indexer, with integer position fallback. @@ -1258,7 +1286,6 @@ def _get_slice_axis(self, slice_obj, axis=0): class _LocIndexer(_LocationIndexer): - """Purely label-location based indexer for selection by label. ``.loc[]`` is primarily label based, but may also be used with a @@ -1317,16 +1344,16 @@ def _has_valid_type(self, key, axis): def error(): if isnull(key): - raise TypeError( - "cannot use label indexing with a null key") + raise TypeError("cannot use label indexing with a null " + "key") raise KeyError("the label [%s] is not in the [%s]" % (key, self.obj._get_axis_name(axis))) try: key = self._convert_scalar_indexer(key, axis) - if not key in ax: + if key not in ax: error() - except (TypeError) as e: + except TypeError as e: # python 3 type errors should be raised if 'unorderable' in str(e): # pragma: no cover @@ -1351,12 +1378,12 @@ def _getitem_axis(self, key, axis=0): # possibly convert a list-like into a nested tuple # but don't convert a list-like of tuples if isinstance(labels, MultiIndex): - if not isinstance(key, tuple) and len(key) > 1 and not isinstance(key[0], tuple): + if (not isinstance(key, tuple) and len(key) > 1 and + not isinstance(key[0], tuple)): key = tuple([key]) # an iterable multi-selection - if not (isinstance(key, tuple) and - isinstance(labels, MultiIndex)): + if not (isinstance(key, tuple) and isinstance(labels, MultiIndex)): if hasattr(key, 'ndim') and key.ndim > 1: raise ValueError('Cannot index with multidimensional key') @@ -1366,7 +1393,7 @@ def _getitem_axis(self, key, axis=0): # nested tuple slicing if is_nested_tuple(key, labels): locs = labels.get_locs(key) - indexer = [ slice(None) ] * self.ndim + indexer = [slice(None)] * self.ndim indexer[axis] = locs return self.obj.iloc[tuple(indexer)] @@ -1376,7 +1403,6 @@ def _getitem_axis(self, key, axis=0): class _iLocIndexer(_LocationIndexer): - """Purely integer-location based indexing for selection by position. ``.iloc[]`` is primarily integer position based (from ``0`` to @@ -1406,10 +1432,9 @@ def _has_valid_type(self, key, axis): if is_bool_indexer(key): if hasattr(key, 'index') and isinstance(key.index, Index): if key.index.inferred_type == 'integer': - raise NotImplementedError( - "iLocation based boolean indexing on an integer type " - "is not available" - ) + raise NotImplementedError("iLocation based boolean " + "indexing on an integer type " + "is not available") raise ValueError("iLocation based boolean indexing cannot use " "an indexable as a mask") return True @@ -1434,9 +1459,9 @@ def _is_valid_integer(self, key, axis): raise IndexError("single positional indexer is out-of-bounds") return True - def _is_valid_list_like(self, key, axis): - # return a boolean if we are a valid list-like (e.g. that we dont' have out-of-bounds values) + # return a boolean if we are a valid list-like (e.g. that we don't + # have out-of-bounds values) # coerce the key to not exceed the maximum size of the index arr = np.array(key) @@ -1456,7 +1481,7 @@ def _getitem_tuple(self, tup): pass retval = self.obj - axis=0 + axis = 0 for i, key in enumerate(tup): if i >= self.obj.ndim: raise IndexingError('Too many indexers') @@ -1468,7 +1493,7 @@ def _getitem_tuple(self, tup): retval = getattr(retval, self.name)._getitem_axis(key, axis=axis) # if the dim was reduced, then pass a lower-dim the next time - if retval.ndim IndexSlices[:, ['A', 'B']] - kinds = tuple(list(compat.string_types) + - [ABCSeries, np.ndarray, Index, list]) + kinds = tuple(list(compat.string_types) + [ABCSeries, np.ndarray, Index, + list]) if isinstance(slice_, kinds): slice_ = IndexSlice[:, slice_] diff --git a/pandas/core/internals.py b/pandas/core/internals.py index b10b1b5771bf7..6e9005395281c 100644 --- a/pandas/core/internals.py +++ b/pandas/core/internals.py @@ -8,15 +8,15 @@ import numpy as np from pandas.core.base import PandasObject -from pandas.core.common import (_possibly_downcast_to_dtype, isnull, - _NS_DTYPE, _TD_DTYPE, ABCSeries, is_list_like, - ABCSparseSeries, _infer_dtype_from_scalar, - is_null_slice, is_dtype_equal, - is_null_datelike_scalar, _maybe_promote, - is_timedelta64_dtype, is_datetime64_dtype, - is_datetime64tz_dtype, is_datetimetz, is_sparse, - array_equivalent, _maybe_convert_string_to_object, - is_categorical, needs_i8_conversion, is_datetimelike_v_numeric, +from pandas.core.common import (_possibly_downcast_to_dtype, isnull, _NS_DTYPE, + _TD_DTYPE, ABCSeries, is_list_like, + _infer_dtype_from_scalar, is_null_slice, + is_dtype_equal, is_null_datelike_scalar, + _maybe_promote, is_timedelta64_dtype, + is_datetime64_dtype, is_datetimetz, is_sparse, + array_equivalent, + _maybe_convert_string_to_object, + is_categorical, is_datetimelike_v_numeric, is_numeric_v_string_like, is_internal_type) from pandas.core.dtypes import DatetimeTZDtype @@ -33,17 +33,14 @@ import pandas.computation.expressions as expressions from pandas.util.decorators import cache_readonly -from pandas.tslib import Timestamp, Timedelta +from pandas.tslib import Timedelta from pandas import compat from pandas.compat import range, map, zip, u -from pandas.tseries.timedeltas import _coerce_scalar_to_timedelta_type - from pandas.lib import BlockPlacement class Block(PandasObject): - """ Canonical n-dimensional unit of homogeneous dtype contained in a pandas data structure @@ -82,9 +79,9 @@ def __init__(self, values, placement, ndim=None, fastpath=False): self.values = values if len(self.mgr_locs) != len(self.values): - raise ValueError('Wrong number of items passed %d,' - ' placement implies %d' % ( - len(self.values), len(self.mgr_locs))) + raise ValueError('Wrong number of items passed %d, placement ' + 'implies %d' % (len(self.values), + len(self.mgr_locs))) @property def _consolidate_key(self): @@ -125,7 +122,8 @@ def external_values(self, dtype=None): def internal_values(self, dtype=None): """ return an internal format, currently just the ndarray - this should be the pure internal API format """ + this should be the pure internal API format + """ return self.values def get_values(self, dtype=None): @@ -141,7 +139,7 @@ def to_dense(self): def to_object_block(self, mgr): """ return myself as an object block """ values = self.get_values(dtype=object) - return self.make_block(values,klass=ObjectBlock) + return self.make_block(values, klass=ObjectBlock) @property def fill_value(self): @@ -153,13 +151,15 @@ def mgr_locs(self): @property def array_dtype(self): - """ the dtype to return if I want to construct this block as an array """ + """ the dtype to return if I want to construct this block as an + array + """ return self.dtype def make_block(self, values, placement=None, ndim=None, **kwargs): """ - Create a new block, with type inference - propogate any values that are not specified + Create a new block, with type inference propogate any values that are + not specified """ if placement is None: placement = self.mgr_locs @@ -168,7 +168,8 @@ def make_block(self, values, placement=None, ndim=None, **kwargs): return make_block(values, placement=placement, ndim=ndim, **kwargs) - def make_block_same_class(self, values, placement=None, fastpath=True, **kwargs): + def make_block_same_class(self, values, placement=None, fastpath=True, + **kwargs): """ Wrap given values in a block of same type as self. """ if placement is None: placement = self.mgr_locs @@ -188,15 +189,13 @@ def __unicode__(self): name = com.pprint_thing(self.__class__.__name__) if self._is_single_block: - result = '%s: %s dtype: %s' % ( - name, len(self), self.dtype) + result = '%s: %s dtype: %s' % (name, len(self), self.dtype) else: shape = ' x '.join([com.pprint_thing(s) for s in self.shape]) - result = '%s: %s, %s, dtype: %s' % ( - name, com.pprint_thing(self.mgr_locs.indexer), shape, - self.dtype) + result = '%s: %s, %s, dtype: %s' % (name, com.pprint_thing( + self.mgr_locs.indexer), shape, self.dtype) return result @@ -226,19 +225,15 @@ def reshape_nd(self, labels, shape, ref_items, mgr=None): return a new block that is transformed to a nd block """ - return _block2d_to_blocknd( - values=self.get_values().T, - placement=self.mgr_locs, - shape=shape, - labels=labels, - ref_items=ref_items) + return _block2d_to_blocknd(values=self.get_values().T, + placement=self.mgr_locs, shape=shape, + labels=labels, ref_items=ref_items) def getitem_block(self, slicer, new_mgr_locs=None): """ Perform __getitem__-like, return result as block. As of now, only supports slices that preserve dimensionality. - """ if new_mgr_locs is None: if isinstance(slicer, tuple): @@ -285,8 +280,7 @@ def reindex_axis(self, indexer, method=None, axis=1, fill_value=None, new_values = com.take_nd(self.values, indexer, axis, fill_value=fill_value, mask_info=mask_info) - return self.make_block(new_values, - fastpath=True) + return self.make_block(new_values, fastpath=True) def get(self, item): loc = self.items.get_loc(item) @@ -313,16 +307,20 @@ def delete(self, loc): self.mgr_locs = self.mgr_locs.delete(loc) def apply(self, func, mgr=None, **kwargs): - """ apply the function to my values; return a block if we are not one """ + """ apply the function to my values; return a block if we are not + one + """ result = func(self.values, **kwargs) if not isinstance(result, Block): result = self.make_block(values=_block_shape(result)) return result - def fillna(self, value, limit=None, inplace=False, downcast=None, mgr=None): - """ fillna on the block with the value. If we fail, then convert to ObjectBlock - and try again """ + def fillna(self, value, limit=None, inplace=False, downcast=None, + mgr=None): + """ fillna on the block with the value. If we fail, then convert to + ObjectBlock and try again + """ if not self._can_hold_na: if inplace: @@ -336,13 +334,14 @@ def fillna(self, value, limit=None, inplace=False, downcast=None, mgr=None): if self.ndim > 2: raise NotImplementedError("number of dimensions for 'fillna' " "is currently limited to 2") - mask[mask.cumsum(self.ndim-1) > limit] = False + mask[mask.cumsum(self.ndim - 1) > limit] = False # fillna, but if we cannot coerce, then try again as an ObjectBlock try: values, _, value, _ = self._try_coerce_args(self.values, value) blocks = self.putmask(mask, value, inplace=inplace) - blocks = [ b.make_block(values=self._try_coerce_result(b.values)) for b in blocks ] + blocks = [b.make_block(values=self._try_coerce_result(b.values)) + for b in blocks] return self._maybe_downcast(blocks, downcast) except (TypeError, ValueError): @@ -366,7 +365,7 @@ def _maybe_downcast(self, blocks, downcast=None): elif downcast is None and (self.is_timedelta or self.is_datetime): return blocks - return _extend_blocks([ b.downcast(downcast) for b in blocks ]) + return _extend_blocks([b.downcast(downcast) for b in blocks]) def downcast(self, dtypes=None, mgr=None): """ try to downcast each item to the dict of dtypes if present """ @@ -385,8 +384,7 @@ def downcast(self, dtypes=None, mgr=None): dtypes = 'infer' nv = _possibly_downcast_to_dtype(values, dtypes) - return self.make_block(nv, - fastpath=True) + return self.make_block(nv, fastpath=True) # ndim > 1 if dtypes is None: @@ -405,7 +403,8 @@ def downcast(self, dtypes=None, mgr=None): dtype = 'infer' else: raise AssertionError("dtypes as dict is not supported yet") - dtype = dtypes.get(item, self._downcast_dtype) + # TODO: This either should be completed or removed + dtype = dtypes.get(item, self._downcast_dtype) # noqa if dtype is None: nv = _block_shape(values[i], ndim=self.ndim) @@ -413,13 +412,12 @@ def downcast(self, dtypes=None, mgr=None): nv = _possibly_downcast_to_dtype(values[i], dtype) nv = _block_shape(nv, ndim=self.ndim) - blocks.append(self.make_block(nv, - fastpath=True, - placement=[rl])) + blocks.append(self.make_block(nv, fastpath=True, placement=[rl])) return blocks - def astype(self, dtype, copy=False, raise_on_error=True, values=None, **kwargs): + def astype(self, dtype, copy=False, raise_on_error=True, values=None, + **kwargs): return self._astype(dtype, copy=copy, raise_on_error=raise_on_error, values=values, **kwargs) @@ -449,7 +447,8 @@ def _astype(self, dtype, copy=False, raise_on_error=True, values=None, # force the copy here if values is None: - if issubclass(dtype.type, (compat.text_type, compat.string_types)): + if issubclass(dtype.type, + (compat.text_type, compat.string_types)): # use native type formatting for datetime/tz/timedelta if self.is_datelike: @@ -466,9 +465,7 @@ def _astype(self, dtype, copy=False, raise_on_error=True, values=None, values = com._astype_nansafe(values.ravel(), dtype, copy=True) values = values.reshape(self.shape) - newb = make_block(values, - placement=self.mgr_locs, - dtype=dtype, + newb = make_block(values, placement=self.mgr_locs, dtype=dtype, klass=klass) except: if raise_on_error is True: @@ -485,9 +482,10 @@ def _astype(self, dtype, copy=False, raise_on_error=True, values=None, return newb def convert(self, copy=True, **kwargs): - """ attempt to coerce any object types to better types - return a copy of the block (if copy = True) - by definition we are not an ObjectBlock here! """ + """ attempt to coerce any object types to better types return a copy + of the block (if copy = True) by definition we are not an ObjectBlock + here! + """ return self.copy() if copy else self @@ -498,8 +496,9 @@ def _try_cast(self, value): raise NotImplementedError() def _try_cast_result(self, result, dtype=None): - """ try to cast the result to our original type, - we may have roundtripped thru object in the mean-time """ + """ try to cast the result to our original type, we may have + roundtripped thru object in the mean-time + """ if dtype is None: dtype = self.dtype @@ -549,7 +548,8 @@ def _try_coerce_and_cast_result(self, result, dtype=None): def _try_fill(self, value): return value - def to_native_types(self, slicer=None, na_rep='nan', quoting=None, **kwargs): + def to_native_types(self, slicer=None, na_rep='nan', quoting=None, + **kwargs): """ convert to our native types format, slicing if desired """ values = self.values @@ -578,14 +578,17 @@ def replace(self, to_replace, value, inplace=False, filter=None, """ replace the to_replace value with value, possible to create new blocks here this is just a call to putmask. regex is not used here. It is used in ObjectBlocks. It is here for API - compatibility.""" + compatibility. + """ original_to_replace = to_replace mask = isnull(self.values) - # try to replace, if we raise an error, convert to ObjectBlock and retry + # try to replace, if we raise an error, convert to ObjectBlock and + # retry try: - values, _, to_replace, _ = self._try_coerce_args(self.values, to_replace) + values, _, to_replace, _ = self._try_coerce_args(self.values, + to_replace) mask = com.mask_missing(values, to_replace) if filter is not None: filtered_out = ~self.mgr_locs.isin(filter) @@ -593,7 +596,8 @@ def replace(self, to_replace, value, inplace=False, filter=None, blocks = self.putmask(mask, value, inplace=inplace) if convert: - blocks = [ b.convert(by_item=True, numeric=False, copy=not inplace) for b in blocks ] + blocks = [b.convert(by_item=True, numeric=False, + copy=not inplace) for b in blocks] return blocks except (TypeError, ValueError): @@ -601,13 +605,9 @@ def replace(self, to_replace, value, inplace=False, filter=None, if not mask.any(): return self if inplace else self.copy() - return self.to_object_block(mgr=mgr).replace(to_replace=original_to_replace, - value=value, - inplace=inplace, - filter=filter, - regex=regex, - convert=convert) - + return self.to_object_block(mgr=mgr).replace( + to_replace=original_to_replace, value=value, inplace=inplace, + filter=filter, regex=regex, convert=convert) def _replace_single(self, *args, **kwargs): """ no-op on a non-ObjectBlock """ @@ -665,7 +665,7 @@ def _is_scalar_indexer(indexer): if arr_value.ndim == 1: if not isinstance(indexer, tuple): indexer = tuple([indexer]) - return all([ np.isscalar(idx) for idx in indexer ]) + return all([np.isscalar(idx) for idx in indexer]) return False def _is_empty_indexer(indexer): @@ -674,7 +674,8 @@ def _is_empty_indexer(indexer): if arr_value.ndim == 1: if not isinstance(indexer, tuple): indexer = tuple([indexer]) - return any(isinstance(idx, np.ndarray) and len(idx) == 0 for idx in indexer) + return any(isinstance(idx, np.ndarray) and len(idx) == 0 + for idx in indexer) return False # empty indexers @@ -682,14 +683,17 @@ def _is_empty_indexer(indexer): if _is_empty_indexer(indexer): pass - # setting a single element for each dim and with a rhs that could be say a list + # setting a single element for each dim and with a rhs that could + # be say a list # GH 6043 elif _is_scalar_indexer(indexer): values[indexer] = value # if we are an exact match (ex-broadcasting), # then use the resultant dtype - elif len(arr_value.shape) and arr_value.shape[0] == values.shape[0] and np.prod(arr_value.shape) == np.prod(values.shape): + elif (len(arr_value.shape) and + arr_value.shape[0] == values.shape[0] and + np.prod(arr_value.shape) == np.prod(values.shape)): values[indexer] = value values = values.astype(arr_value.dtype) @@ -703,23 +707,22 @@ def _is_empty_indexer(indexer): else: dtype = 'infer' values = self._try_coerce_and_cast_result(values, dtype) - block = self.make_block(transf(values), - fastpath=True) + block = self.make_block(transf(values), fastpath=True) # may have to soft convert_objects here if block.is_object and not self.is_object: block = block.convert(numeric=False) return block - except (ValueError, TypeError) as detail: + except (ValueError, TypeError): raise - except Exception as detail: + except Exception: pass return [self] - def putmask(self, mask, new, align=True, inplace=False, - axis=0, transpose=False, mgr=None): + def putmask(self, mask, new, align=True, inplace=False, axis=0, + transpose=False, mgr=None): """ putmask the data to the block; it is possible that we may create a new dtype of block @@ -758,12 +761,12 @@ def putmask(self, mask, new, align=True, inplace=False, new = self._try_cast(new) - # If the default repeat behavior in np.putmask would go in the wrong - # direction, then explictly repeat and reshape new instead + # If the default repeat behavior in np.putmask would go in the + # wrong direction, then explictly repeat and reshape new instead if getattr(new, 'ndim', 0) >= 1: if self.ndim - 1 == new.ndim and axis == 1: - new = np.repeat(new, new_values.shape[-1]).reshape( - self.shape) + new = np.repeat( + new, new_values.shape[-1]).reshape(self.shape) new = new.astype(new_values.dtype) np.putmask(new_values, mask, new) @@ -810,15 +813,13 @@ def putmask(self, mask, new, align=True, inplace=False, # Put back the dimension that was taken from it and make # a block out of the result. block = self.make_block(values=nv[np.newaxis], - placement=[ref_loc], - fastpath=True) + placement=[ref_loc], fastpath=True) new_blocks.append(block) else: nv = _putmask_smart(new_values, mask, new) - new_blocks.append(self.make_block(values=nv, - fastpath=True)) + new_blocks.append(self.make_block(values=nv, fastpath=True)) return new_blocks @@ -828,14 +829,12 @@ def putmask(self, mask, new, align=True, inplace=False, if transpose: new_values = new_values.T - return [self.make_block(new_values, - fastpath=True)] - - def interpolate(self, method='pad', axis=0, index=None, - values=None, inplace=False, limit=None, - limit_direction='forward', - fill_value=None, coerce=False, downcast=None, mgr=None, **kwargs): + return [self.make_block(new_values, fastpath=True)] + def interpolate(self, method='pad', axis=0, index=None, values=None, + inplace=False, limit=None, limit_direction='forward', + fill_value=None, coerce=False, downcast=None, mgr=None, + **kwargs): def check_int_bool(self, inplace): # Only FloatBlocks will contain NaNs. # timedelta subclasses IntBlock @@ -855,14 +854,11 @@ def check_int_bool(self, inplace): r = check_int_bool(self, inplace) if r is not None: return r - return self._interpolate_with_fill(method=m, - axis=axis, - inplace=inplace, - limit=limit, + return self._interpolate_with_fill(method=m, axis=axis, + inplace=inplace, limit=limit, fill_value=fill_value, coerce=coerce, - downcast=downcast, - mgr=mgr) + downcast=downcast, mgr=mgr) # try an interp method try: m = mis._clean_interp_method(method, **kwargs) @@ -873,17 +869,11 @@ def check_int_bool(self, inplace): r = check_int_bool(self, inplace) if r is not None: return r - return self._interpolate(method=m, - index=index, - values=values, - axis=axis, - limit=limit, + return self._interpolate(method=m, index=index, values=values, + axis=axis, limit=limit, limit_direction=limit_direction, - fill_value=fill_value, - inplace=inplace, - downcast=downcast, - mgr=mgr, - **kwargs) + fill_value=fill_value, inplace=inplace, + downcast=downcast, mgr=mgr, **kwargs) raise ValueError("invalid method '{0}' to interpolate.".format(method)) @@ -904,23 +894,18 @@ def _interpolate_with_fill(self, method='pad', axis=0, inplace=False, values = self.values if inplace else self.values.copy() values, _, fill_value, _ = self._try_coerce_args(values, fill_value) values = self._try_operate(values) - values = mis.interpolate_2d(values, - method=method, - axis=axis, - limit=limit, - fill_value=fill_value, + values = mis.interpolate_2d(values, method=method, axis=axis, + limit=limit, fill_value=fill_value, dtype=self.dtype) values = self._try_coerce_result(values) - blocks = [self.make_block(values, - klass=self.__class__, - fastpath=True)] + blocks = [self.make_block(values, klass=self.__class__, fastpath=True)] return self._maybe_downcast(blocks, downcast) def _interpolate(self, method=None, index=None, values=None, fill_value=None, axis=0, limit=None, - limit_direction='forward', - inplace=False, downcast=None, mgr=None, **kwargs): + limit_direction='forward', inplace=False, downcast=None, + mgr=None, **kwargs): """ interpolate using scipy wrappers """ data = self.values if inplace else self.values.copy() @@ -953,8 +938,7 @@ def func(x): # interp each column independently interp_values = np.apply_along_axis(func, axis, data) - blocks = [self.make_block(interp_values, - klass=self.__class__, + blocks = [self.make_block(interp_values, klass=self.__class__, fastpath=True)] return self._maybe_downcast(blocks, downcast) @@ -999,8 +983,7 @@ def take_nd(self, indexer, axis, new_mgr_locs=None, fill_tuple=None): def diff(self, n, axis=1, mgr=None): """ return block for the diff of the values """ new_values = com.diff(self.values, n, axis=axis) - return [self.make_block(values=new_values, - fastpath=True)] + return [self.make_block(values=new_values, fastpath=True)] def shift(self, periods, axis=0, mgr=None): """ shift the block by periods, possibly upcast """ @@ -1016,21 +999,21 @@ def shift(self, periods, axis=0, mgr=None): axis = new_values.ndim - axis - 1 if np.prod(new_values.shape): - new_values = np.roll(new_values, com._ensure_platform_int(periods), axis=axis) + new_values = np.roll(new_values, com._ensure_platform_int(periods), + axis=axis) - axis_indexer = [ slice(None) ] * self.ndim + axis_indexer = [slice(None)] * self.ndim if periods > 0: - axis_indexer[axis] = slice(None,periods) + axis_indexer[axis] = slice(None, periods) else: - axis_indexer[axis] = slice(periods,None) + axis_indexer[axis] = slice(periods, None) new_values[tuple(axis_indexer)] = fill_value # restore original order if f_ordered: new_values = new_values.T - return [self.make_block(new_values, - fastpath=True)] + return [self.make_block(new_values, fastpath=True)] def eval(self, func, other, raise_on_error=True, try_cast=False, mgr=None): """ @@ -1072,7 +1055,8 @@ def eval(self, func, other, raise_on_error=True, try_cast=False, mgr=None): transf = (lambda x: x.T) if is_transposed else (lambda x: x) # coerce/transpose the args if needed - values, values_mask, other, other_mask = self._try_coerce_args(transf(values), other) + values, values_mask, other, other_mask = self._try_coerce_args( + transf(values), other) # get the result, may need to transpose the other def get_result(other): @@ -1090,13 +1074,13 @@ def get_result(other): # mask if needed if isinstance(values_mask, np.ndarray) and values_mask.any(): - result = result.astype('float64',copy=False) + result = result.astype('float64', copy=False) result[values_mask] = np.nan if other_mask is True: - result = result.astype('float64',copy=False) + result = result.astype('float64', copy=False) result[:] = np.nan elif isinstance(other_mask, np.ndarray) and other_mask.any(): - result = result.astype('float64',copy=False) + result = result.astype('float64', copy=False) result[other_mask.ravel()] = np.nan return self._try_coerce_result(result) @@ -1105,8 +1089,8 @@ def get_result(other): def handle_error(): if raise_on_error: - raise TypeError('Could not operate %s with block values %s' - % (repr(other), str(detail))) + raise TypeError('Could not operate %s with block values %s' % + (repr(other), str(detail))) else: # return the values result = np.empty(values.shape, dtype='O') @@ -1135,8 +1119,8 @@ def handle_error(): raise ValueError('Invalid broadcasting comparison [%s] ' 'with block values' % repr(other)) - raise TypeError('Could not compare [%s] with block values' - % repr(other)) + raise TypeError('Could not compare [%s] with block values' % + repr(other)) # transpose if needed result = transf(result) @@ -1145,8 +1129,7 @@ def handle_error(): if try_cast: result = self._try_cast_result(result) - return [self.make_block(result, - fastpath=True,)] + return [self.make_block(result, fastpath=True, )] def where(self, other, cond, align=True, raise_on_error=True, try_cast=False, axis=0, transpose=False, mgr=None): @@ -1183,10 +1166,11 @@ def where(self, other, cond, align=True, raise_on_error=True, # explictly reshape other instead if getattr(other, 'ndim', 0) >= 1: if values.ndim - 1 == other.ndim and axis == 1: - other = other.reshape(tuple(other.shape + (1,))) + other = other.reshape(tuple(other.shape + (1, ))) if not hasattr(cond, 'shape'): - raise ValueError("where must have a condition that is ndarray like") + raise ValueError("where must have a condition that is ndarray " + "like") other = _maybe_convert_string_to_object(other) @@ -1195,11 +1179,11 @@ def func(cond, values, other): if cond.ravel().all(): return values - values, values_mask, other, other_mask = self._try_coerce_args(values, other) + values, values_mask, other, other_mask = self._try_coerce_args( + values, other) try: - return self._try_coerce_result( - expressions.where(cond, values, other, raise_on_error=True) - ) + return self._try_coerce_result(expressions.where( + cond, values, other, raise_on_error=True)) except Exception as detail: if raise_on_error: raise TypeError('Could not operate [%s] with block values ' @@ -1233,15 +1217,16 @@ def func(cond, values, other): result_blocks = [] for m in [mask, ~mask]: if m.any(): - r = self._try_cast_result( - result.take(m.nonzero()[0], axis=axis)) - result_blocks.append(self.make_block(r.T, - placement=self.mgr_locs[m])) + r = self._try_cast_result(result.take(m.nonzero()[0], + axis=axis)) + result_blocks.append( + self.make_block(r.T, placement=self.mgr_locs[m])) return result_blocks def equals(self, other): - if self.dtype != other.dtype or self.shape != other.shape: return False + if self.dtype != other.dtype or self.shape != other.shape: + return False return array_equivalent(self.values, other.values) @@ -1252,8 +1237,7 @@ class NonConsolidatableMixIn(object): _validate_ndim = False _holder = None - def __init__(self, values, placement, - ndim=None, fastpath=False, **kwargs): + def __init__(self, values, placement, ndim=None, fastpath=False, **kwargs): # Placement must be converted to BlockPlacement via property setter # before ndim logic, because placement may be a slice which doesn't @@ -1312,11 +1296,11 @@ def get(self, item): else: return self.values - def putmask(self, mask, new, align=True, inplace=False, - axis=0, transpose=False, mgr=None): + def putmask(self, mask, new, align=True, inplace=False, axis=0, + transpose=False, mgr=None): """ - putmask the data to the block; we must be a single block and not generate - other blocks + putmask the data to the block; we must be a single block and not + generate other blocks return the resulting block @@ -1358,7 +1342,8 @@ class FloatOrComplexBlock(NumericBlock): __slots__ = () def equals(self, other): - if self.dtype != other.dtype or self.shape != other.shape: return False + if self.dtype != other.dtype or self.shape != other.shape: + return False left, right = self.values, other.values return ((left == right) | (np.isnan(left) & np.isnan(right))).all() @@ -1372,10 +1357,11 @@ def _can_hold_element(self, element): if is_list_like(element): element = np.array(element) tipo = element.dtype.type - return issubclass(tipo, (np.floating, np.integer)) and not issubclass( - tipo, (np.datetime64, np.timedelta64)) - return isinstance(element, (float, int, np.float_, np.int_)) and not isinstance( - element, (bool, np.bool_, datetime, timedelta, np.datetime64, np.timedelta64)) + return (issubclass(tipo, (np.floating, np.integer)) and + not issubclass(tipo, (np.datetime64, np.timedelta64))) + return (isinstance(element, (float, int, np.float_, np.int_)) and + not isinstance(element, (bool, np.bool_, datetime, timedelta, + np.datetime64, np.timedelta64))) def _try_cast(self, element): try: @@ -1383,8 +1369,8 @@ def _try_cast(self, element): except: # pragma: no cover return element - def to_native_types(self, slicer=None, na_rep='', float_format=None, decimal='.', - quoting=None, **kwargs): + def to_native_types(self, slicer=None, na_rep='', float_format=None, + decimal='.', quoting=None, **kwargs): """ convert to our native types format, slicing if desired """ values = self.values @@ -1411,8 +1397,10 @@ class ComplexBlock(FloatOrComplexBlock): def _can_hold_element(self, element): if is_list_like(element): element = np.array(element) - return issubclass(element.dtype.type, (np.floating, np.integer, np.complexfloating)) - return (isinstance(element, (float, int, complex, np.float_, np.int_)) and + return issubclass(element.dtype.type, + (np.floating, np.integer, np.complexfloating)) + return (isinstance(element, + (float, int, complex, np.float_, np.int_)) and not isinstance(bool, np.bool_)) def _try_cast(self, element): @@ -1434,7 +1422,8 @@ def _can_hold_element(self, element): if is_list_like(element): element = np.array(element) tipo = element.dtype.type - return issubclass(tipo, np.integer) and not issubclass(tipo, (np.datetime64, np.timedelta64)) + return (issubclass(tipo, np.integer) and + not issubclass(tipo, (np.datetime64, np.timedelta64))) return com.is_integer(element) def _try_cast(self, element): @@ -1462,7 +1451,7 @@ def fillna(self, value, **kwargs): # allow filling with integers to be # interpreted as seconds if not isinstance(value, np.timedelta64) and com.is_integer(value): - value = Timedelta(value,unit='s') + value = Timedelta(value, unit='s') return super(TimeDeltaBlock, self).fillna(value, **kwargs) def _try_coerce_args(self, values, other): @@ -1499,7 +1488,7 @@ def _try_coerce_args(self, values, other): other = Timedelta(other).value elif isinstance(other, np.ndarray): other_mask = isnull(other) - other = other.astype('i8',copy=False).view('i8') + other = other.astype('i8', copy=False).view('i8') else: # scalar other = Timedelta(other) @@ -1526,7 +1515,8 @@ def _try_coerce_result(self, result): def should_store(self, value): return issubclass(value.dtype.type, np.timedelta64) - def to_native_types(self, slicer=None, na_rep=None, quoting=None, **kwargs): + def to_native_types(self, slicer=None, na_rep=None, quoting=None, + **kwargs): """ convert to our native types format, slicing if desired """ values = self.values @@ -1540,7 +1530,7 @@ def to_native_types(self, slicer=None, na_rep=None, quoting=None, **kwargs): rvalues[mask] = na_rep imask = (~mask).ravel() - #### FIXME #### + # FIXME: # should use the core.format.Timedelta64Formatter here # to figure what format to pass to the Timedelta # e.g. to not show the decimals say @@ -1549,14 +1539,14 @@ def to_native_types(self, slicer=None, na_rep=None, quoting=None, **kwargs): dtype=object) return rvalues - def get_values(self, dtype=None): # return object dtypes as Timedelta if dtype == object: - return lib.map_infer(self.values.ravel(), lib.Timedelta - ).reshape(self.values.shape) + return lib.map_infer(self.values.ravel(), + lib.Timedelta).reshape(self.values.shape) return self.values + class BoolBlock(NumericBlock): __slots__ = () is_bool = True @@ -1592,13 +1582,12 @@ class ObjectBlock(Block): is_object = True _can_hold_na = True - def __init__(self, values, ndim=2, fastpath=False, - placement=None, **kwargs): + def __init__(self, values, ndim=2, fastpath=False, placement=None, + **kwargs): if issubclass(values.dtype.type, compat.string_types): values = np.array(values, dtype=object) - super(ObjectBlock, self).__init__(values, ndim=ndim, - fastpath=fastpath, + super(ObjectBlock, self).__init__(values, ndim=ndim, fastpath=fastpath, placement=placement, **kwargs) @property @@ -1610,17 +1599,16 @@ def is_bool(self): # TODO: Refactor when convert_objects is removed since there will be 1 path def convert(self, *args, **kwargs): - """ attempt to coerce any object types to better types - return a copy of the block (if copy = True) - by definition we ARE an ObjectBlock!!!!! + """ attempt to coerce any object types to better types return a copy of + the block (if copy = True) by definition we ARE an ObjectBlock!!!!! - can return multiple blocks! - """ + can return multiple blocks! + """ if args: raise NotImplementedError by_item = True if 'by_item' not in kwargs else kwargs['by_item'] - new_inputs = ['coerce','datetime','numeric','timedelta'] + new_inputs = ['coerce', 'datetime', 'numeric', 'timedelta'] new_style = False for kw in new_inputs: new_style |= kw in kwargs @@ -1630,7 +1618,8 @@ def convert(self, *args, **kwargs): fn_inputs = new_inputs else: fn = convert._possibly_convert_objects - fn_inputs = ['convert_dates','convert_numeric','convert_timedeltas'] + fn_inputs = ['convert_dates', 'convert_numeric', + 'convert_timedeltas'] fn_inputs += ['copy'] fn_kwargs = {} @@ -1651,8 +1640,10 @@ def convert(self, *args, **kwargs): blocks.append(newb) else: - values = fn(self.values.ravel(), **fn_kwargs).reshape(self.values.shape) - blocks.append(make_block(values, ndim=self.ndim, placement=self.mgr_locs)) + values = fn( + self.values.ravel(), **fn_kwargs).reshape(self.values.shape) + blocks.append(make_block(values, ndim=self.ndim, + placement=self.mgr_locs)) return blocks @@ -1680,18 +1671,18 @@ def set(self, locs, values, check=False): # see GH6171 new_shape = list(values.shape) new_shape[0] = len(self.items) - self.values = np.empty(tuple(new_shape),dtype=self.dtype) + self.values = np.empty(tuple(new_shape), dtype=self.dtype) self.values.fill(np.nan) self.values[locs] = values - def _maybe_downcast(self, blocks, downcast=None): if downcast is not None: return blocks # split and convert the blocks - return _extend_blocks([ b.convert(datetime=True, numeric=False) for b in blocks ]) + return _extend_blocks([b.convert(datetime=True, numeric=False) + for b in blocks]) def _can_hold_element(self, element): return True @@ -1701,8 +1692,9 @@ def _try_cast(self, element): def should_store(self, value): return not (issubclass(value.dtype.type, - (np.integer, np.floating, np.complexfloating, - np.datetime64, np.bool_)) or is_internal_type(value)) + (np.integer, np.floating, np.complexfloating, + np.datetime64, np.bool_)) or + is_internal_type(value)) def replace(self, to_replace, value, inplace=False, filter=None, regex=False, convert=True, mgr=None): @@ -1715,9 +1707,9 @@ def replace(self, to_replace, value, inplace=False, filter=None, blocks = [self] if not either_list and com.is_re(to_replace): - return self._replace_single(to_replace, value, - inplace=inplace, filter=filter, - regex=True, convert=convert, mgr=mgr) + return self._replace_single(to_replace, value, inplace=inplace, + filter=filter, regex=True, + convert=convert, mgr=mgr) elif not (either_list or regex): return super(ObjectBlock, self).replace(to_replace, value, inplace=inplace, @@ -1738,17 +1730,16 @@ def replace(self, to_replace, value, inplace=False, filter=None, for to_rep in to_replace: result_blocks = [] for b in blocks: - result = b._replace_single(to_rep, value, - inplace=inplace, + result = b._replace_single(to_rep, value, inplace=inplace, filter=filter, regex=regex, convert=convert, mgr=mgr) result_blocks = _extend_blocks(result, result_blocks) blocks = result_blocks return result_blocks - return self._replace_single(to_replace, value, - inplace=inplace, filter=filter, - convert=convert, regex=regex, mgr=mgr) + return self._replace_single(to_replace, value, inplace=inplace, + filter=filter, convert=convert, + regex=regex, mgr=mgr) def _replace_single(self, to_replace, value, inplace=False, filter=None, regex=False, convert=True, mgr=None): @@ -1785,8 +1776,7 @@ def _replace_single(self, to_replace, value, inplace=False, filter=None, # the superclass method -> to_replace is some kind of object return super(ObjectBlock, self).replace(to_replace, value, inplace=inplace, - filter=filter, - regex=regex, + filter=filter, regex=regex, mgr=mgr) new_values = self.values if inplace else self.values.copy() @@ -1794,6 +1784,7 @@ def _replace_single(self, to_replace, value, inplace=False, filter=None, # deal with replacing values with objects (strings) that match but # whose replacement is not a string (numeric, nan, object) if isnull(value) or not isinstance(value, compat.string_types): + def re_replacer(s): try: return value if rx.search(s) is not None else s @@ -1820,10 +1811,11 @@ def re_replacer(s): # convert block = self.make_block(new_values) if convert: - block = block.convert(by_item=True,numeric=False) + block = block.convert(by_item=True, numeric=False) return block + class CategoricalBlock(NonConsolidatableMixIn, ObjectBlock): __slots__ = () is_categorical = True @@ -1831,13 +1823,12 @@ class CategoricalBlock(NonConsolidatableMixIn, ObjectBlock): _can_hold_na = True _holder = Categorical - def __init__(self, values, placement, - fastpath=False, **kwargs): + def __init__(self, values, placement, fastpath=False, **kwargs): # coerce to categorical if we can super(CategoricalBlock, self).__init__(maybe_to_categorical(values), - fastpath=True, placement=placement, - **kwargs) + fastpath=True, + placement=placement, **kwargs) @property def is_view(self): @@ -1852,7 +1843,9 @@ def convert(self, copy=True, **kwargs): @property def array_dtype(self): - """ the dtype to return if I want to construct this block as an array """ + """ the dtype to return if I want to construct this block as an + array + """ return np.object_ def _slice(self, slicer): @@ -1862,7 +1855,8 @@ def _slice(self, slicer): # return same dims as we currently have return self.values._slice(slicer) - def fillna(self, value, limit=None, inplace=False, downcast=None, mgr=None): + def fillna(self, value, limit=None, inplace=False, downcast=None, + mgr=None): # we may need to upcast our fill to match our dtype if limit is not None: raise NotImplementedError("specifying a limit for 'fillna' has " @@ -1873,14 +1867,14 @@ def fillna(self, value, limit=None, inplace=False, downcast=None, mgr=None): limit=limit)) return [self.make_block(values=values)] - def interpolate(self, method='pad', axis=0, inplace=False, - limit=None, fill_value=None, **kwargs): + def interpolate(self, method='pad', axis=0, inplace=False, limit=None, + fill_value=None, **kwargs): values = self.values if inplace else self.values.copy() - return self.make_block_same_class(values=values.fillna(fill_value=fill_value, - method=method, - limit=limit), - placement=self.mgr_locs) + return self.make_block_same_class( + values=values.fillna(fill_value=fill_value, method=method, + limit=limit), + placement=self.mgr_locs) def shift(self, periods, axis=0, mgr=None): return self.make_block_same_class(values=self.values.shift(periods), @@ -1889,7 +1883,6 @@ def shift(self, periods, axis=0, mgr=None): def take_nd(self, indexer, axis=0, new_mgr_locs=None, fill_tuple=None): """ Take values according to indexer and return them as a block.bb - """ if fill_tuple is None: fill_value = None @@ -1939,21 +1932,20 @@ def to_native_types(self, slicer=None, na_rep='', quoting=None, **kwargs): values[mask] = na_rep # we are expected to return a 2-d ndarray - return values.reshape(1,len(values)) + return values.reshape(1, len(values)) + class DatetimeBlock(Block): __slots__ = () is_datetime = True _can_hold_na = True - def __init__(self, values, placement, - fastpath=False, **kwargs): + def __init__(self, values, placement, fastpath=False, **kwargs): if values.dtype != _NS_DTYPE: values = tslib.cast_to_nanoseconds(values) - super(DatetimeBlock, self).__init__(values, - fastpath=True, placement=placement, - **kwargs) + super(DatetimeBlock, self).__init__(values, fastpath=True, + placement=placement, **kwargs) def _astype(self, dtype, mgr=None, **kwargs): """ @@ -1966,7 +1958,7 @@ def _astype(self, dtype, mgr=None, **kwargs): dtype = DatetimeTZDtype(dtype) values = self.values - if getattr(values,'tz',None) is None: + if getattr(values, 'tz', None) is None: values = DatetimeIndex(values).tz_localize('UTC') values = values.tz_convert(dtype.tz) return self.make_block(values) @@ -1974,13 +1966,11 @@ def _astype(self, dtype, mgr=None, **kwargs): # delegate return super(DatetimeBlock, self)._astype(dtype=dtype, **kwargs) - def _can_hold_element(self, element): if is_list_like(element): element = np.array(element) return element.dtype == _NS_DTYPE or element.dtype == np.int64 - return (com.is_integer(element) or - isinstance(element, datetime) or + return (com.is_integer(element) or isinstance(element, datetime) or isnull(element)) def _try_cast(self, element): @@ -2021,8 +2011,9 @@ def _try_coerce_args(self, values, other): other_mask = True elif isinstance(other, (datetime, np.datetime64, date)): other = lib.Timestamp(other) - if getattr(other,'tz') is not None: - raise TypeError("cannot coerce a Timestamp with a tz on a naive Block") + if getattr(other, 'tz') is not None: + raise TypeError("cannot coerce a Timestamp with a tz on a " + "naive Block") other_mask = isnull(other) other = other.asm8.view('i8') elif hasattr(other, 'dtype') and com.is_integer_dtype(other): @@ -2032,7 +2023,7 @@ def _try_coerce_args(self, values, other): other = np.asarray(other) other_mask = isnull(other) - other = other.astype('i8',copy=False).view('i8') + other = other.astype('i8', copy=False).view('i8') except ValueError: # coercion issues @@ -2065,14 +2056,14 @@ def to_native_types(self, slicer=None, na_rep=None, date_format=None, from pandas.core.format import _get_format_datetime64_from_values format = _get_format_datetime64_from_values(values, date_format) - result = tslib.format_array_from_datetime(values.view('i8').ravel(), - tz=getattr(self.values,'tz',None), - format=format, - na_rep=na_rep).reshape(values.shape) + result = tslib.format_array_from_datetime( + values.view('i8').ravel(), tz=getattr(self.values, 'tz', None), + format=format, na_rep=na_rep).reshape(values.shape) return np.atleast_2d(result) def should_store(self, value): - return issubclass(value.dtype.type, np.datetime64) and not is_datetimetz(value) + return (issubclass(value.dtype.type, np.datetime64) and + not is_datetimetz(value)) def set(self, locs, values, check=False): """ @@ -2091,28 +2082,27 @@ def set(self, locs, values, check=False): def get_values(self, dtype=None): # return object dtype as Timestamps if dtype == object: - return lib.map_infer(self.values.ravel(), lib.Timestamp)\ - .reshape(self.values.shape) + return lib.map_infer( + self.values.ravel(), lib.Timestamp).reshape(self.values.shape) return self.values + class DatetimeTZBlock(NonConsolidatableMixIn, DatetimeBlock): """ implement a datetime64 block with a tz attribute """ __slots__ = () _holder = DatetimeIndex is_datetimetz = True - def __init__(self, values, placement, ndim=2, - **kwargs): + def __init__(self, values, placement, ndim=2, **kwargs): if not isinstance(values, self._holder): values = self._holder(values) if values.tz is None: raise ValueError("cannot create a DatetimeTZBlock without a tz") - super(DatetimeTZBlock, self).__init__(values, - placement=placement, - ndim=ndim, - **kwargs) + super(DatetimeTZBlock, self).__init__(values, placement=placement, + ndim=ndim, **kwargs) + def copy(self, deep=True, mgr=None): """ copy constructor """ values = self.values @@ -2121,15 +2111,17 @@ def copy(self, deep=True, mgr=None): return self.make_block_same_class(values) def external_values(self): - """ we internally represent the data as a DatetimeIndex, but for external - compat with ndarray, export as a ndarray of Timestamps """ + """ we internally represent the data as a DatetimeIndex, but for + external compat with ndarray, export as a ndarray of Timestamps + """ return self.values.astype('datetime64[ns]').values def get_values(self, dtype=None): # return object dtype as Timestamps with the zones if dtype == object: - return lib.map_infer(self.values.ravel(), lambda x: lib.Timestamp(x,tz=self.values.tz))\ - .reshape(self.values.shape) + f = lambda x: lib.Timestamp(x, tz=self.values.tz) + return lib.map_infer( + self.values.ravel(), f).reshape(self.values.shape) return self.values def to_object_block(self, mgr): @@ -2142,9 +2134,9 @@ def to_object_block(self, mgr): values = self.get_values(dtype=object) kwargs = {} if mgr.ndim > 1: - values = _block_shape(values,ndim=mgr.ndim) + values = _block_shape(values, ndim=mgr.ndim) kwargs['ndim'] = mgr.ndim - kwargs['placement']=[0] + kwargs['placement'] = [0] return self.make_block(values, klass=ObjectBlock, **kwargs) def replace(self, *args, **kwargs): @@ -2216,7 +2208,8 @@ def _try_coerce_result(self, result): def shift(self, periods, axis=0, mgr=None): """ shift the block by periods """ - ### think about moving this to the DatetimeIndex. This is a non-freq (number of periods) shift ### + # think about moving this to the DatetimeIndex. This is a non-freq + # (number of periods) shift ### N = len(self) indexer = np.zeros(N, dtype=int) @@ -2233,8 +2226,10 @@ def shift(self, periods, axis=0, mgr=None): else: new_values[periods:] = tslib.iNaT - new_values = DatetimeIndex(new_values,tz=self.values.tz) - return [self.make_block_same_class(new_values, placement=self.mgr_locs)] + new_values = DatetimeIndex(new_values, tz=self.values.tz) + return [self.make_block_same_class(new_values, + placement=self.mgr_locs)] + class SparseBlock(NonConsolidatableMixIn, Block): """ implement as a list of sparse arrays of the same dtype """ @@ -2256,7 +2251,7 @@ def itemsize(self): @property def fill_value(self): - #return np.nan + # return np.nan return self.values.fill_value @fill_value.setter @@ -2301,9 +2296,9 @@ def copy(self, deep=True, mgr=None): kind=self.kind, copy=deep, placement=self.mgr_locs) - def make_block_same_class(self, values, placement, - sparse_index=None, kind=None, dtype=None, - fill_value=None, copy=False, fastpath=True, **kwargs): + def make_block_same_class(self, values, placement, sparse_index=None, + kind=None, dtype=None, fill_value=None, + copy=False, fastpath=True, **kwargs): """ return a new block """ if dtype is None: dtype = self.dtype @@ -2332,19 +2327,19 @@ def make_block_same_class(self, values, placement, new_values = SparseArray(values, sparse_index=sparse_index, kind=kind or self.kind, dtype=dtype, fill_value=fill_value, copy=copy) - return self.make_block(new_values, - fastpath=fastpath, + return self.make_block(new_values, fastpath=fastpath, placement=placement) - def interpolate(self, method='pad', axis=0, inplace=False, - limit=None, fill_value=None, **kwargs): + def interpolate(self, method='pad', axis=0, inplace=False, limit=None, + fill_value=None, **kwargs): - values = mis.interpolate_2d( - self.values.to_dense(), method, axis, limit, fill_value) + values = mis.interpolate_2d(self.values.to_dense(), method, axis, + limit, fill_value) return self.make_block_same_class(values=values, placement=self.mgr_locs) - def fillna(self, value, limit=None, inplace=False, downcast=None, mgr=None): + def fillna(self, value, limit=None, inplace=False, downcast=None, + mgr=None): # we may need to upcast our fill to match our dtype if limit is not None: raise NotImplementedError("specifying a limit for 'fillna' has " @@ -2372,7 +2367,8 @@ def shift(self, periods, axis=0, mgr=None): new_values[:periods] = fill_value else: new_values[periods:] = fill_value - return [self.make_block_same_class(new_values, placement=self.mgr_locs)] + return [self.make_block_same_class(new_values, + placement=self.mgr_locs)] def reindex_axis(self, indexer, method=None, axis=1, fill_value=None, limit=None, mask_info=None): @@ -2396,11 +2392,11 @@ def sparse_reindex(self, new_index): values = values.sp_index.to_int_index().reindex( values.sp_values.astype('float64'), values.fill_value, new_index) return self.make_block_same_class(values, sparse_index=new_index, - placement=self.mgr_locs) + placement=self.mgr_locs) -def make_block(values, placement, klass=None, ndim=None, - dtype=None, fastpath=False): +def make_block(values, placement, klass=None, ndim=None, dtype=None, + fastpath=False): if klass is None: dtype = dtype or values.dtype vtype = dtype.type @@ -2410,15 +2406,15 @@ def make_block(values, placement, klass=None, ndim=None, elif issubclass(vtype, np.floating): klass = FloatBlock elif (issubclass(vtype, np.integer) and - issubclass(vtype, np.timedelta64)): + issubclass(vtype, np.timedelta64)): klass = TimeDeltaBlock elif (issubclass(vtype, np.integer) and - not issubclass(vtype, np.datetime64)): + not issubclass(vtype, np.datetime64)): klass = IntBlock elif dtype == np.bool_: klass = BoolBlock elif issubclass(vtype, np.datetime64): - if hasattr(values,'tz'): + if hasattr(values, 'tz'): klass = DatetimeTZBlock else: klass = DatetimeBlock @@ -2431,15 +2427,12 @@ def make_block(values, placement, klass=None, ndim=None, else: klass = ObjectBlock - return klass(values, ndim=ndim, fastpath=fastpath, - placement=placement) - + return klass(values, ndim=ndim, fastpath=fastpath, placement=placement) # TODO: flexible with index=None and/or items=None class BlockManager(PandasObject): - """ Core internal data structure to implement DataFrame, Series, Panel, etc. @@ -2501,12 +2494,13 @@ def __init__(self, blocks, axes, do_integrity_check=True, fastpath=True): for block in blocks: if block.is_sparse: if len(block.mgr_locs) != 1: - raise AssertionError("Sparse block refers to multiple items") + raise AssertionError("Sparse block refers to multiple " + "items") else: if self.ndim != block.ndim: - raise AssertionError(('Number of Block dimensions (%d) must ' - 'equal number of axes (%d)') - % (block.ndim, self.ndim)) + raise AssertionError('Number of Block dimensions (%d) ' + 'must equal number of axes (%d)' % + (block.ndim, self.ndim)) if do_integrity_check: self._verify_integrity() @@ -2518,9 +2512,8 @@ def __init__(self, blocks, axes, do_integrity_check=True, fastpath=True): def make_empty(self, axes=None): """ return an empty BlockManager with the items axis of len 0 """ if axes is None: - axes = [_ensure_index([])] + [ - _ensure_index(a) for a in self.axes[1:] - ] + axes = [_ensure_index([])] + [_ensure_index(a) + for a in self.axes[1:]] # preserve dtype if possible if self.ndim == 1: @@ -2550,7 +2543,8 @@ def set_axis(self, axis, new_labels): if new_len != old_len: raise ValueError('Length mismatch: Expected axis has %d elements, ' - 'new values have %d elements' % (old_len, new_len)) + 'new values have %d elements' % + (old_len, new_len)) self.axes[axis] = new_labels @@ -2612,6 +2606,7 @@ def _rebuild_blknos_and_blklocs(self): # make items read only for now def _get_items(self): return self.axes[0] + items = property(fget=_get_items) def _get_counts(self, f): @@ -2645,8 +2640,7 @@ def __getstate__(self): extra_state = { '0.14.1': { 'axes': axes_array, - 'blocks': [dict(values=b.values, - mgr_locs=b.mgr_locs.indexer) + 'blocks': [dict(values=b.values, mgr_locs=b.mgr_locs.indexer) for b in self.blocks] } } @@ -2662,13 +2656,12 @@ def unpickle_block(values, mgr_locs): values = values.astype('M8[ns]') return make_block(values, placement=mgr_locs) - if (isinstance(state, tuple) and len(state) >= 4 - and '0.14.1' in state[3]): + if (isinstance(state, tuple) and len(state) >= 4 and + '0.14.1' in state[3]): state = state[3]['0.14.1'] self.axes = [_ensure_index(ax) for ax in state['axes']] - self.blocks = tuple( - unpickle_block(b['values'], b['mgr_locs']) - for b in state['blocks']) + self.blocks = tuple(unpickle_block(b['values'], b['mgr_locs']) + for b in state['blocks']) else: # discard anything after 3rd, support beta pickling format for a # little while longer @@ -2724,10 +2717,11 @@ def _verify_integrity(self): if len(self.items) != tot_items: raise AssertionError('Number of manager items must equal union of ' 'block items\n# manager items: {0}, # ' - 'tot_items: {1}'.format(len(self.items), - tot_items)) + 'tot_items: {1}'.format( + len(self.items), tot_items)) - def apply(self, f, axes=None, filter=None, do_integrity_check=False, consolidate=True, **kwargs): + def apply(self, f, axes=None, filter=None, do_integrity_check=False, + consolidate=True, **kwargs): """ iterate over the blocks, collect and create a new block manager @@ -2737,8 +2731,10 @@ def apply(self, f, axes=None, filter=None, do_integrity_check=False, consolidate axes : optional (if not supplied, use self.axes) filter : list, if supplied, only call the block if the filter is in the block - do_integrity_check : boolean, default False. Do the block manager integrity check - consolidate: boolean, default True. Join together blocks having same dtype + do_integrity_check : boolean, default False. Do the block manager + integrity check + consolidate: boolean, default True. Join together blocks having same + dtype Returns ------- @@ -2783,7 +2779,8 @@ def apply(self, f, axes=None, filter=None, do_integrity_check=False, consolidate else: align_keys = [] - aligned_args = dict((k, kwargs[k]) for k in align_keys + aligned_args = dict((k, kwargs[k]) + for k in align_keys if hasattr(kwargs[k], 'reindex_axis')) for b in self.blocks: @@ -2850,7 +2847,8 @@ def convert(self, **kwargs): def replace(self, **kwargs): return self.apply('replace', **kwargs) - def replace_list(self, src_list, dest_list, inplace=False, regex=False, mgr=None): + def replace_list(self, src_list, dest_list, inplace=False, regex=False, + mgr=None): """ do a list replace """ if mgr is None: @@ -2864,6 +2862,7 @@ def comp(s): return isnull(values) return _possibly_compare(values, getattr(s, 'asm8', s), operator.eq) + masks = [comp(s) for i, s in enumerate(src_list)] result_blocks = [] @@ -2876,8 +2875,8 @@ def comp(s): new_rb = [] for b in rb: if b.dtype == np.object_: - result = b.replace(s, d, inplace=inplace, - regex=regex, mgr=mgr) + result = b.replace(s, d, inplace=inplace, regex=regex, + mgr=mgr) new_rb = _extend_blocks(result, new_rb) else: # get our mask for this element, sized to this @@ -2970,7 +2969,8 @@ def combine(self, blocks, copy=True): return self.make_empty() # FIXME: optimization potential - indexer = np.sort(np.concatenate([b.mgr_locs.as_array for b in blocks])) + indexer = np.sort(np.concatenate([b.mgr_locs.as_array + for b in blocks])) inv_indexer = lib.get_reverse_indexer(indexer, self.shape[0]) new_items = self.items.take(indexer) @@ -3033,7 +3033,7 @@ def copy(self, deep=True, mgr=None): copy = lambda ax: ax.copy(deep=True) else: copy = lambda ax: ax.view() - new_axes = [ copy(ax) for ax in self.axes] + new_axes = [copy(ax) for ax in self.axes] else: new_axes = list(self.axes) return self.apply('copy', axes=new_axes, deep=deep, @@ -3088,8 +3088,8 @@ def _interleave(self): def xs(self, key, axis=1, copy=True, takeable=False): if axis < 1: - raise AssertionError('Can only take xs across axis >= 1, got %d' - % axis) + raise AssertionError('Can only take xs across axis >= 1, got %d' % + axis) # take by position if takeable: @@ -3122,8 +3122,10 @@ def xs(self, key, axis=1, copy=True, takeable=False): vals = block.values[slicer] if copy: vals = vals.copy() - new_blocks = [make_block(values=vals, placement=block.mgr_locs, - klass=block.__class__, fastpath=True,)] + new_blocks = [make_block(values=vals, + placement=block.mgr_locs, + klass=block.__class__, + fastpath=True, )] return self.__class__(new_blocks, new_axes) @@ -3208,14 +3210,14 @@ def get(self, item, fastpath=True): indexer = self.items.get_indexer_for([item]) return self.reindex_indexer(new_axis=self.items[indexer], - indexer=indexer, axis=0, allow_dups=True) + indexer=indexer, axis=0, + allow_dups=True) def iget(self, i, fastpath=True): """ Return the data as a SingleBlockManager if fastpath=True and possible Otherwise return as a ndarray - """ block = self.blocks[self._blknos[i]] values = block.iget(self._blklocs[i]) @@ -3223,19 +3225,17 @@ def iget(self, i, fastpath=True): return values # fastpath shortcut for select a single-dim from a 2-dim BM - return SingleBlockManager([ block.make_block_same_class(values, - placement=slice(0, len(values)), - ndim=1, - fastpath=True) ], - self.axes[1]) - + return SingleBlockManager( + [block.make_block_same_class(values, + placement=slice(0, len(values)), + ndim=1, fastpath=True)], + self.axes[1]) def get_scalar(self, tup): """ Retrieve single item """ - full_loc = list(ax.get_loc(x) - for ax, x in zip(self.axes, tup)) + full_loc = list(ax.get_loc(x) for ax, x in zip(self.axes, tup)) blk = self.blocks[self._blknos[full_loc[0]]] values = blk.values @@ -3297,6 +3297,7 @@ def set(self, item, value, check=False): # categorical/spares/datetimetz if value_is_internal_type: + def value_getitem(placement): return value else: @@ -3306,8 +3307,10 @@ def value_getitem(placement): def value_getitem(placement): return value else: + def value_getitem(placement): return value[placement.indexer] + if value.shape[1:] != self.shape[1:]: raise AssertionError('Shape of new values must be compatible ' 'with manager shape') @@ -3416,9 +3419,8 @@ def insert(self, loc, item, value, allow_duplicates=False): # insert to the axis; this could possibly raise a TypeError new_axis = self.items.insert(loc, item) - block = make_block(values=value, - ndim=self.ndim, - placement=slice(loc, loc+1)) + block = make_block(values=value, ndim=self.ndim, + placement=slice(loc, loc + 1)) for blkno, count in _fast_count_smallints(self._blknos[loc:]): blk = self.blocks[blkno] @@ -3453,8 +3455,8 @@ def reindex_axis(self, new_index, axis, method=None, limit=None, Conform block manager to new index. """ new_index = _ensure_index(new_index) - new_index, indexer = self.axes[axis].reindex( - new_index, method=method, limit=limit) + new_index, indexer = self.axes[axis].reindex(new_index, method=method, + limit=limit) return self.reindex_indexer(new_index, indexer, axis=axis, fill_value=fill_value, copy=copy) @@ -3491,13 +3493,12 @@ def reindex_indexer(self, new_axis, indexer, axis, fill_value=None, raise IndexError("Requested axis not found in manager") if axis == 0: - new_blocks = self._slice_take_blocks_ax0( - indexer, fill_tuple=(fill_value,)) + new_blocks = self._slice_take_blocks_ax0(indexer, + fill_tuple=(fill_value,)) else: - new_blocks = [blk.take_nd(indexer, axis=axis, - fill_tuple=(fill_value if fill_value is not None else - blk.fill_value,)) - for blk in self.blocks] + new_blocks = [blk.take_nd(indexer, axis=axis, fill_tuple=( + fill_value if fill_value is not None else blk.fill_value,)) + for blk in self.blocks] new_axes = list(self.axes) new_axes[axis] = new_axis @@ -3524,12 +3525,11 @@ def _slice_take_blocks_ax0(self, slice_or_indexer, fill_tuple=None): blk = self.blocks[0] if sl_type in ('slice', 'mask'): - return [blk.getitem_block(slobj, - new_mgr_locs=slice(0, sllen))] + return [blk.getitem_block(slobj, new_mgr_locs=slice(0, sllen))] elif not allow_fill or self.ndim == 1: if allow_fill and fill_tuple[0] is None: _, fill_value = com._maybe_promote(blk.dtype) - fill_tuple = (fill_value,) + fill_tuple = (fill_value, ) return [blk.take_nd(slobj, axis=0, new_mgr_locs=slice(0, sllen), @@ -3556,24 +3556,25 @@ def _slice_take_blocks_ax0(self, slice_or_indexer, fill_tuple=None): # If we've got here, fill_tuple was not None. fill_value = fill_tuple[0] - blocks.append(self._make_na_block( - placement=mgr_locs, fill_value=fill_value)) + blocks.append(self._make_na_block(placement=mgr_locs, + fill_value=fill_value)) else: blk = self.blocks[blkno] # Otherwise, slicing along items axis is necessary. if not blk._can_consolidate: - # A non-consolidatable block, it's easy, because there's only one item - # and each mgr loc is a copy of that single item. + # A non-consolidatable block, it's easy, because there's + # only one item and each mgr loc is a copy of that single + # item. for mgr_loc in mgr_locs: newblk = blk.copy(deep=True) newblk.mgr_locs = slice(mgr_loc, mgr_loc + 1) blocks.append(newblk) else: - blocks.append(blk.take_nd( - blklocs[mgr_locs.indexer], axis=0, - new_mgr_locs=mgr_locs, fill_tuple=None)) + blocks.append(blk.take_nd(blklocs[mgr_locs.indexer], + axis=0, new_mgr_locs=mgr_locs, + fill_tuple=None)) return blocks @@ -3595,9 +3596,10 @@ def take(self, indexer, axis=1, verify=True, convert=True): Take items along any axis. """ self._consolidate_inplace() - indexer = np.arange(indexer.start, indexer.stop, indexer.step, - dtype='int64') if isinstance(indexer, slice) \ - else np.asanyarray(indexer, dtype='int64') + indexer = (np.arange(indexer.start, indexer.stop, indexer.step, + dtype='int64') + if isinstance(indexer, slice) + else np.asanyarray(indexer, dtype='int64')) n = self.shape[axis] if convert: @@ -3620,8 +3622,7 @@ def merge(self, other, lsuffix='', rsuffix=''): right=other.items, rsuffix=rsuffix) new_items = _concat_indexes([l, r]) - new_blocks = [blk.copy(deep=False) - for blk in self.blocks] + new_blocks = [blk.copy(deep=False) for blk in self.blocks] offset = self.shape[0] for blk in other.blocks: @@ -3639,8 +3640,8 @@ def _is_indexed_like(self, other): Check all axes except items """ if self.ndim != other.ndim: - raise AssertionError(('Number of dimensions must agree ' - 'got %d and %d') % (self.ndim, other.ndim)) + raise AssertionError('Number of dimensions must agree ' + 'got %d and %d' % (self.ndim, other.ndim)) for ax, oax in zip(self.axes[1:], other.axes[1:]): if not ax.equals(oax): return False @@ -3650,7 +3651,7 @@ def equals(self, other): self_axes, other_axes = self.axes, other.axes if len(self_axes) != len(other_axes): return False - if not all (ax1.equals(ax2) for ax1, ax2 in zip(self_axes, other_axes)): + if not all(ax1.equals(ax2) for ax1, ax2 in zip(self_axes, other_axes)): return False self._consolidate_inplace() other._consolidate_inplace() @@ -3666,8 +3667,8 @@ def canonicalize(block): self_blocks = sorted(self.blocks, key=canonicalize) other_blocks = sorted(other.blocks, key=canonicalize) - return all(block.equals(oblock) for block, oblock in - zip(self_blocks, other_blocks)) + return all(block.equals(oblock) + for block, oblock in zip(self_blocks, other_blocks)) class SingleBlockManager(BlockManager): @@ -3682,8 +3683,8 @@ def __init__(self, block, axis, do_integrity_check=False, fastpath=False): if isinstance(axis, list): if len(axis) != 1: - raise ValueError( - "cannot create SingleBlockManager with more than 1 axis") + raise ValueError("cannot create SingleBlockManager with more " + "than 1 axis") axis = axis[0] # passed from constructor, single block, single axis @@ -3716,9 +3717,8 @@ def __init__(self, block, axis, do_integrity_check=False, fastpath=False): block = block[0] if not isinstance(block, Block): - block = make_block(block, - placement=slice(0, len(axis)), - ndim=1, fastpath=True) + block = make_block(block, placement=slice(0, len(axis)), ndim=1, + fastpath=True) self.blocks = [block] @@ -3754,8 +3754,7 @@ def reindex(self, new_axis, indexer=None, method=None, fill_value=None, else: fill_value = np.nan - new_values = com.take_1d(values, indexer, - fill_value=fill_value) + new_values = com.take_1d(values, indexer, fill_value=fill_value) # fill if needed if method is not None or limit is not None: @@ -3820,7 +3819,7 @@ def internal_values(self): def get_values(self): """ return a dense type view """ - return np.array(self._block.to_dense(),copy=False) + return np.array(self._block.to_dense(), copy=False) @property def itemsize(self): @@ -3864,7 +3863,7 @@ def construction_error(tot_items, block_shape, axes, e=None): if passed == implied and e is not None: raise e raise ValueError("Shape of passed values is {0}, indices imply {1}".format( - passed,implied)) + passed, implied)) def create_block_manager_from_blocks(blocks, axes): @@ -3874,8 +3873,8 @@ def create_block_manager_from_blocks(blocks, axes): if not len(blocks[0]): blocks = [] else: - # It's OK if a single block is passed as values, its placement is - # basically "all items", but if there're many, don't bother + # It's OK if a single block is passed as values, its placement + # is basically "all items", but if there're many, don't bother # converting, it's an error anyway. blocks = [make_block(values=blocks[0], placement=slice(0, len(axes[0])))] @@ -3897,7 +3896,7 @@ def create_block_manager_from_arrays(arrays, names, axes): mgr = BlockManager(blocks, axes) mgr._consolidate_inplace() return mgr - except (ValueError) as e: + except ValueError as e: construction_error(len(arrays), arrays[0].shape, axes, e) @@ -3949,7 +3948,7 @@ def form_blocks(arrays, names, axes): elif issubclass(v.dtype.type, np.integer): if v.dtype == np.uint64: # HACK #2355 definite overflow - if (v > 2 ** 63 - 1).any(): + if (v > 2**63 - 1).any(): object_items.append((i, k, v)) continue int_items.append((i, k, v)) @@ -3974,26 +3973,23 @@ def form_blocks(arrays, names, axes): blocks.extend(int_blocks) if len(datetime_items): - datetime_blocks = _simple_blockify( - datetime_items, _NS_DTYPE) + datetime_blocks = _simple_blockify(datetime_items, _NS_DTYPE) blocks.extend(datetime_blocks) if len(datetime_tz_items): - dttz_blocks = [ make_block(array, - klass=DatetimeTZBlock, - fastpath=True, - placement=[i], - ) for i, names, array in datetime_tz_items ] + dttz_blocks = [make_block(array, + klass=DatetimeTZBlock, + fastpath=True, + placement=[i], ) + for i, names, array in datetime_tz_items] blocks.extend(dttz_blocks) if len(bool_items): - bool_blocks = _simple_blockify( - bool_items, np.bool_) + bool_blocks = _simple_blockify(bool_items, np.bool_) blocks.extend(bool_blocks) if len(object_items) > 0: - object_blocks = _simple_blockify( - object_items, np.object_) + object_blocks = _simple_blockify(object_items, np.object_) blocks.extend(object_blocks) if len(sparse_items) > 0: @@ -4001,11 +3997,9 @@ def form_blocks(arrays, names, axes): blocks.extend(sparse_blocks) if len(cat_items) > 0: - cat_blocks = [ make_block(array, - klass=CategoricalBlock, - fastpath=True, - placement=[i] - ) for i, names, array in cat_items ] + cat_blocks = [make_block(array, klass=CategoricalBlock, fastpath=True, + placement=[i]) + for i, names, array in cat_items] blocks.extend(cat_blocks) if len(extra_locs): @@ -4044,8 +4038,7 @@ def _multi_blockify(tuples, dtype=None): new_blocks = [] for dtype, tup_block in grouper: - values, placement = _stack_arrays( - list(tup_block), dtype) + values, placement = _stack_arrays(list(tup_block), dtype) block = make_block(values, placement=placement) new_blocks.append(block) @@ -4061,9 +4054,8 @@ def _sparse_blockify(tuples, dtype=None): new_blocks = [] for i, names, array in tuples: array = _maybe_to_sparse(array) - block = make_block( - array, klass=SparseBlock, fastpath=True, - placement=[i]) + block = make_block(array, klass=SparseBlock, fastpath=True, + placement=[i]) new_blocks.append(block) return new_blocks @@ -4121,18 +4113,17 @@ def _lcd_dtype(l): have_dt64_tz = len(counts[DatetimeTZBlock]) > 0 have_td64 = len(counts[TimeDeltaBlock]) > 0 have_cat = len(counts[CategoricalBlock]) > 0 - have_sparse = len(counts[SparseBlock]) > 0 + # TODO: have_sparse is not used + have_sparse = len(counts[SparseBlock]) > 0 # noqa have_numeric = have_float or have_complex or have_int has_non_numeric = have_dt64 or have_dt64_tz or have_td64 or have_cat if (have_object or - (have_bool and (have_numeric or have_dt64 or have_dt64_tz or have_td64)) or - (have_numeric and has_non_numeric) or - have_cat or - have_dt64 or - have_dt64_tz or - have_td64): + (have_bool and + (have_numeric or have_dt64 or have_dt64_tz or have_td64)) or + (have_numeric and has_non_numeric) or have_cat or have_dt64 or + have_dt64_tz or have_td64): return np.dtype(object) elif have_bool: return np.dtype(bool) @@ -4197,8 +4188,7 @@ def _merge_blocks(blocks, dtype=None, _can_consolidate=True): new_values = new_values[argsort] new_mgr_locs = new_mgr_locs[argsort] - return make_block(new_values, - fastpath=True, placement=new_mgr_locs) + return make_block(new_values, fastpath=True, placement=new_mgr_locs) # no merge return blocks @@ -4220,12 +4210,13 @@ def _extend_blocks(result, blocks=None): blocks.append(result) return blocks + def _block_shape(values, ndim=1, shape=None): """ guarantee the shape of the values to be at least 1 d """ if values.ndim <= ndim: if shape is None: shape = values.shape - values = values.reshape(tuple((1,) + shape)) + values = values.reshape(tuple((1, ) + shape)) return values @@ -4310,6 +4301,7 @@ def _factor_indexer(shape, labels): return com._ensure_platform_int( np.sum(np.array(labels).T * np.append(mult, [1]), axis=1).T) + def _get_blkno_placements(blknos, blk_count, group=True): """ @@ -4391,7 +4383,7 @@ def _putmask_smart(v, m, n): # n should be the length of the mask or a scalar here if not is_list_like(n): n = np.array([n] * len(m)) - elif isinstance(n, np.ndarray) and n.ndim == 0: # numpy scalar + elif isinstance(n, np.ndarray) and n.ndim == 0: # numpy scalar n = np.repeat(np.array(n, ndmin=1), len(m)) # see if we are only masking values that if putted @@ -4434,9 +4426,9 @@ def concatenate_block_managers(mgrs_indexers, axes, concat_axis, copy): copy : bool """ - concat_plan = combine_concat_plans([get_mgr_concatenation_plan(mgr, indexers) - for mgr, indexers in mgrs_indexers], - concat_axis) + concat_plan = combine_concat_plans( + [get_mgr_concatenation_plan(mgr, indexers) + for mgr, indexers in mgrs_indexers], concat_axis) blocks = [make_block(concatenate_join_units(join_units, concat_axis, copy=copy), @@ -4550,6 +4542,7 @@ def concatenate_join_units(join_units, concat_axis, copy): return concat_values + def get_mgr_concatenation_plan(mgr, indexers): """ Construct concatenation plan for given block manager and indexers. @@ -4603,17 +4596,18 @@ def get_mgr_concatenation_plan(mgr, indexers): blk = mgr.blocks[blkno] ax0_blk_indexer = blklocs[placements.indexer] - unit_no_ax0_reindexing = ( - len(placements) == len(blk.mgr_locs) and - # Fastpath detection of join unit not needing to reindex its - # block: no ax0 reindexing took place and block placement was - # sequential before. - ((ax0_indexer is None - and blk.mgr_locs.is_slice_like - and blk.mgr_locs.as_slice.step == 1) or - # Slow-ish detection: all indexer locs are sequential (and - # length match is checked above). - (np.diff(ax0_blk_indexer) == 1).all())) + unit_no_ax0_reindexing = (len(placements) == len(blk.mgr_locs) and + # Fastpath detection of join unit not + # needing to reindex its block: no ax0 + # reindexing took place and block + # placement was sequential before. + ((ax0_indexer is None and + blk.mgr_locs.is_slice_like and + blk.mgr_locs.as_slice.step == 1) or + # Slow-ish detection: all indexer locs + # are sequential (and length match is + # checked above). + (np.diff(ax0_blk_indexer) == 1).all())) # Omit indexer if no item reindexing is required. if unit_no_ax0_reindexing: @@ -4652,6 +4646,7 @@ def combine_concat_plans(plans, concat_axis): else: num_ended = [0] + def _next_or_none(seq): retval = next(seq, None) if retval is None: @@ -4723,14 +4718,14 @@ class JoinUnit(object): def __init__(self, block, shape, indexers=None): # Passing shape explicitly is required for cases when block is None. if indexers is None: - indexers = {} + indexers = {} self.block = block self.indexers = indexers self.shape = shape def __repr__(self): - return '%s(%r, %s)' % (self.__class__.__name__, - self.block, self.indexers) + return '%s(%r, %s)' % (self.__class__.__name__, self.block, + self.indexers) @cache_readonly def needs_filling(self): @@ -4773,7 +4768,7 @@ def is_null(self): total_len = values_flat.shape[0] chunk_len = max(total_len // 40, 1000) for i in range(0, total_len, chunk_len): - if not isnull(values_flat[i: i + chunk_len]).all(): + if not isnull(values_flat[i:i + chunk_len]).all(): return False return True @@ -4787,7 +4782,8 @@ def get_reindexed_values(self, empty_dtype, upcasted_na): else: fill_value = upcasted_na - if self.is_null and not getattr(self.block,'is_categorical',None): + if self.is_null and not getattr(self.block, 'is_categorical', + None): missing_arr = np.empty(self.shape, dtype=empty_dtype) if np.prod(self.shape): # NumPy 1.6 workaround: this statement gets strange if all diff --git a/pandas/core/missing.py b/pandas/core/missing.py index f1143ad808b91..86640cffc136e 100644 --- a/pandas/core/missing.py +++ b/pandas/core/missing.py @@ -2,11 +2,8 @@ Routines for filling missing data """ -from functools import partial - import numpy as np -import pandas as pd import pandas.core.common as com import pandas.algos as algos import pandas.lib as lib @@ -28,8 +25,8 @@ def _clean_fill_method(method, allow_nearest=False): valid_methods.append('nearest') expecting = 'pad (ffill), backfill (bfill) or nearest' if method not in valid_methods: - msg = ('Invalid fill method. Expecting %s. Got %s' - % (expecting, method)) + msg = ('Invalid fill method. Expecting %s. Got %s' % + (expecting, method)) raise ValueError(msg) return method @@ -37,9 +34,8 @@ def _clean_fill_method(method, allow_nearest=False): def _clean_interp_method(method, **kwargs): order = kwargs.get('order') valid = ['linear', 'time', 'index', 'values', 'nearest', 'zero', 'slinear', - 'quadratic', 'cubic', 'barycentric', 'polynomial', - 'krogh', 'piecewise_polynomial', - 'pchip', 'spline'] + 'quadratic', 'cubic', 'barycentric', 'polynomial', 'krogh', + 'piecewise_polynomial', 'pchip', 'spline'] if method in ('spline', 'polynomial') and order is None: raise ValueError("You must specify the order of the spline or " "polynomial.") @@ -50,8 +46,8 @@ def _clean_interp_method(method, **kwargs): def interpolate_1d(xvalues, yvalues, method='linear', limit=None, - limit_direction='forward', - fill_value=None, bounds_error=False, order=None, **kwargs): + limit_direction='forward', fill_value=None, + bounds_error=False, order=None, **kwargs): """ Logic for the 1-d interpolation. The result should be 1-d, inputs xvalues and yvalues will each be 1-d arrays of the same length. @@ -76,7 +72,7 @@ def interpolate_1d(xvalues, yvalues, method='linear', limit=None, if method == 'time': if not getattr(xvalues, 'is_all_dates', None): - # if not issubclass(xvalues.dtype.type, np.datetime64): + # if not issubclass(xvalues.dtype.type, np.datetime64): raise ValueError('time-weighted interpolation only works ' 'on Series or DataFrames with a ' 'DatetimeIndex') @@ -91,22 +87,21 @@ def _interp_limit(invalid, fw_limit, bw_limit): valid_limit_directions = ['forward', 'backward', 'both'] limit_direction = limit_direction.lower() if limit_direction not in valid_limit_directions: - msg = 'Invalid limit_direction: expecting one of %r, got %r.' % ( - valid_limit_directions, limit_direction) - raise ValueError(msg) + raise ValueError('Invalid limit_direction: expecting one of %r, got ' + '%r.' % (valid_limit_directions, limit_direction)) from pandas import Series ys = Series(yvalues) start_nans = set(range(ys.first_valid_index())) end_nans = set(range(1 + ys.last_valid_index(), len(valid))) - # This is a list of the indexes in the series whose yvalue is currently NaN, - # but whose interpolated yvalue will be overwritten with NaN after computing - # the interpolation. For each index in this list, one of these conditions is - # true of the corresponding NaN in the yvalues: + # This is a list of the indexes in the series whose yvalue is currently + # NaN, but whose interpolated yvalue will be overwritten with NaN after + # computing the interpolation. For each index in this list, one of these + # conditions is true of the corresponding NaN in the yvalues: # - # a) It is one of a chain of NaNs at the beginning of the series, and either - # limit is not specified or limit_direction is 'forward'. + # a) It is one of a chain of NaNs at the beginning of the series, and + # either limit is not specified or limit_direction is 'forward'. # b) It is one of a chain of NaNs at the end of the series, and limit is # specified and limit_direction is 'backward' or 'both'. # c) Limit is nonzero and it is further than limit from the nearest non-NaN @@ -118,9 +113,11 @@ def _interp_limit(invalid, fw_limit, bw_limit): if limit: if limit_direction == 'forward': - violate_limit = sorted(start_nans | set(_interp_limit(invalid, limit, 0))) + violate_limit = sorted(start_nans | set(_interp_limit(invalid, + limit, 0))) if limit_direction == 'backward': - violate_limit = sorted(end_nans | set(_interp_limit(invalid, 0, limit))) + violate_limit = sorted(end_nans | set(_interp_limit(invalid, 0, + limit))) if limit_direction == 'both': violate_limit = sorted(_interp_limit(invalid, limit, limit)) @@ -150,10 +147,13 @@ def _interp_limit(invalid, fw_limit, bw_limit): # hack for DatetimeIndex, #1646 if issubclass(inds.dtype.type, np.datetime64): inds = inds.view(np.int64) - result[invalid] = _interpolate_scipy_wrapper( - inds[valid], yvalues[valid], inds[invalid], method=method, - fill_value=fill_value, - bounds_error=bounds_error, order=order, **kwargs) + result[invalid] = _interpolate_scipy_wrapper(inds[valid], + yvalues[valid], + inds[invalid], + method=method, + fill_value=fill_value, + bounds_error=bounds_error, + order=order, **kwargs) result[violate_limit] = np.nan return result @@ -167,7 +167,8 @@ def _interpolate_scipy_wrapper(x, y, new_x, method, fill_value=None, """ try: from scipy import interpolate - from pandas import DatetimeIndex + # TODO: Why is DatetimeIndex being imported here? + from pandas import DatetimeIndex # noqa except ImportError: raise ImportError('{0} interpolation requires Scipy'.format(method)) @@ -219,7 +220,8 @@ def _interpolate_scipy_wrapper(x, y, new_x, method, fill_value=None, return new_y -def interpolate_2d(values, method='pad', axis=0, limit=None, fill_value=None, dtype=None): +def interpolate_2d(values, method='pad', axis=0, limit=None, fill_value=None, + dtype=None): """ perform an actual interpolation of values, values will be make 2-d if needed fills inplace, returns the result """ @@ -232,7 +234,7 @@ def interpolate_2d(values, method='pad', axis=0, limit=None, fill_value=None, dt if axis != 0: # pragma: no cover raise AssertionError("cannot interpolate on a ndim == 1 with " "axis != 0") - values = values.reshape(tuple((1,) + values.shape)) + values = values.reshape(tuple((1, ) + values.shape)) if fill_value is None: mask = None @@ -241,9 +243,11 @@ def interpolate_2d(values, method='pad', axis=0, limit=None, fill_value=None, dt method = _clean_fill_method(method) if method == 'pad': - values = transf(pad_2d(transf(values), limit=limit, mask=mask, dtype=dtype)) + values = transf(pad_2d( + transf(values), limit=limit, mask=mask, dtype=dtype)) else: - values = transf(backfill_2d(transf(values), limit=limit, mask=mask, dtype=dtype)) + values = transf(backfill_2d( + transf(values), limit=limit, mask=mask, dtype=dtype)) # reshape back if ndim == 1: @@ -256,13 +260,13 @@ def _interp_wrapper(f, wrap_dtype, na_override=None): def wrapper(arr, mask, limit=None): view = arr.view(wrap_dtype) f(view, mask, limit=limit) + return wrapper _pad_1d_datetime = _interp_wrapper(algos.pad_inplace_int64, np.int64) _pad_2d_datetime = _interp_wrapper(algos.pad_2d_inplace_int64, np.int64) -_backfill_1d_datetime = _interp_wrapper(algos.backfill_inplace_int64, - np.int64) +_backfill_1d_datetime = _interp_wrapper(algos.backfill_inplace_int64, np.int64) _backfill_2d_datetime = _interp_wrapper(algos.backfill_2d_inplace_int64, np.int64)