From 8a9994aa3f28cac87d1629c8463e9f4631608826 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 11 Feb 2020 19:43:08 -0800 Subject: [PATCH 1/6] getitem_lowerdim cleanups --- pandas/core/indexing.py | 67 +++++++++++++++-------------------------- 1 file changed, 25 insertions(+), 42 deletions(-) diff --git a/pandas/core/indexing.py b/pandas/core/indexing.py index 44b3c318366d2..3395ffa9219a9 100755 --- a/pandas/core/indexing.py +++ b/pandas/core/indexing.py @@ -718,6 +718,18 @@ def _handle_lowerdim_multi_index_axis0(self, tup: Tuple): return None + def _getitem_tuple_same_dim(self, tup: Tuple): + retval = self.obj + for i, key in enumerate(tup): + if com.is_null_slice(key): + continue + + retval = getattr(retval, self.name)._getitem_axis(key, axis=i) + # We should never have retval.ndim < self.ndim, as that should + # be handled by the _getitem_lowerdim call above. + + return retval + def _getitem_lowerdim(self, tup: Tuple): # we can directly get the axis result since the axis is specified @@ -742,11 +754,15 @@ def _getitem_lowerdim(self, tup: Tuple): raise IndexingError("Too many indexers. handle elsewhere") for i, key in enumerate(tup): - if is_label_like(key) or isinstance(key, tuple): + if is_label_like(key): + # We don't need to check for tuples here because those are + # caught by the _is_nested_tuple_indexer check above. section = self._getitem_axis(key, axis=i) - # we have yielded a scalar ? - if not is_list_like_indexer(section): + if not hasattr(section, "ndim"): + # We should never get here, because _getitem_lowerdim + # is only called after a check for is_scalar_access, + # which this would be. return section elif section.ndim == self.ndim: @@ -755,18 +771,11 @@ def _getitem_lowerdim(self, tup: Tuple): new_key = tup[:i] + (_NS,) + tup[i + 1 :] else: + # Note: the section.ndim == self.ndim check above + # rules out having DataFrame here, so we dont need to worry + # about transposing. new_key = tup[:i] + tup[i + 1 :] - # unfortunately need an odious kludge here because of - # DataFrame transposing convention - if ( - isinstance(section, ABCDataFrame) - and i > 0 - and len(new_key) == 2 - ): - a, b = new_key - new_key = b, a - if len(new_key) == 1: new_key = new_key[0] @@ -774,7 +783,7 @@ def _getitem_lowerdim(self, tup: Tuple): # slice returns a new object. if com.is_null_slice(new_key): return section - # This is an elided recursive call to iloc/loc/etc' + # This is an elided recursive call to iloc/loc' return getattr(section, self.name)[new_key] raise IndexingError("not applicable") @@ -1056,15 +1065,7 @@ def _getitem_tuple(self, tup: Tuple): if self._multi_take_opportunity(tup): return self._multi_take(tup) - # no shortcut needed - retval = self.obj - for i, key in enumerate(tup): - if com.is_null_slice(key): - continue - - retval = getattr(retval, self.name)._getitem_axis(key, axis=i) - - return retval + return self._getitem_tuple_same_dim(tup) def _getitem_axis(self, key, axis: int): key = item_from_zerodim(key) @@ -1475,25 +1476,7 @@ def _getitem_tuple(self, tup: Tuple): except IndexingError: pass - retval = self.obj - axis = 0 - for i, key in enumerate(tup): - if com.is_null_slice(key): - axis += 1 - continue - - retval = getattr(retval, self.name)._getitem_axis(key, axis=axis) - - # if the dim was reduced, then pass a lower-dim the next time - if retval.ndim < self.ndim: - # TODO: this is never reached in tests; can we confirm that - # it is impossible? - axis -= 1 - - # try to get for the next axis - axis += 1 - - return retval + return self._getitem_tuple_same_dim(tup) def _get_list_axis(self, key, axis: int): """ From 0336d11b6ddd47771b290af6d1663f124f7ff452 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Wed, 12 Feb 2020 09:53:08 -0800 Subject: [PATCH 2/6] simplify get_label --- pandas/core/indexing.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/pandas/core/indexing.py b/pandas/core/indexing.py index 3395ffa9219a9..2bc17371af04a 100755 --- a/pandas/core/indexing.py +++ b/pandas/core/indexing.py @@ -579,15 +579,7 @@ def __call__(self, axis=None): return new_self def _get_label(self, label, axis: int): - if self.ndim == 1: - # for perf reasons we want to try _xs first - # as its basically direct indexing - # but will fail when the index is not present - # see GH5667 - return self.obj._xs(label, axis=axis) - elif isinstance(label, tuple) and isinstance(label[axis], slice): - raise IndexingError("no slices here, handle elsewhere") - + # GH#5667 this will fail if the label is not present in the axis. return self.obj._xs(label, axis=axis) def _get_setitem_indexer(self, key): From 2eb50d17f5bdc5646f5c1858aab1000f65dd95d1 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Mon, 17 Feb 2020 18:19:52 -0800 Subject: [PATCH 3/6] CLN: indexing --- pandas/core/indexing.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/pandas/core/indexing.py b/pandas/core/indexing.py index 2c335d69d68a1..1a6c0e569e1d5 100755 --- a/pandas/core/indexing.py +++ b/pandas/core/indexing.py @@ -1325,7 +1325,7 @@ def _validate_read_indexer( # We (temporarily) allow for some missing keys with .loc, except in # some cases (e.g. setting) in which "raise_missing" will be False - if not (self.name == "loc" and not raise_missing): + if raise_missing: not_found = list(set(key) - set(ax)) raise KeyError(f"{not_found} not in index") @@ -1390,10 +1390,7 @@ def _validate_key(self, key, axis: int): else: raise ValueError(f"Can only index by location with a [{self._valid_types}]") - def _has_valid_setitem_indexer(self, indexer): - self._has_valid_positional_setitem_indexer(indexer) - - def _has_valid_positional_setitem_indexer(self, indexer) -> bool: + def _has_valid_setitem_indexer(self, indexer) -> bool: """ Validate that a positional indexer cannot enlarge its target will raise if needed, does not modify the indexer externally. @@ -1602,7 +1599,7 @@ def _setitem_with_indexer(self, indexer, value): # this correctly sets the dtype and avoids cache issues # essentially this separates out the block that is needed # to possibly be modified - if self.ndim > 1 and i == self.obj._info_axis_number: + if self.ndim > 1 and i == info_axis: # add the new item, and set the value # must have all defined axes if we have a scalar From 2e3a3cd7df0b28376341764d9421f68e43935d99 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 18 Feb 2020 11:45:20 -0800 Subject: [PATCH 4/6] CLN: indexing comments --- pandas/core/indexing.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/pandas/core/indexing.py b/pandas/core/indexing.py index 1a6c0e569e1d5..2d00578eda8c3 100755 --- a/pandas/core/indexing.py +++ b/pandas/core/indexing.py @@ -1400,7 +1400,7 @@ def _has_valid_setitem_indexer(self, indexer) -> bool: bool """ if isinstance(indexer, dict): - raise IndexError(f"{self.name} cannot enlarge its target object") + raise IndexError("iloc cannot enlarge its target object") else: if not isinstance(indexer, tuple): indexer = _tuplify(self.ndim, indexer) @@ -1413,11 +1413,9 @@ def _has_valid_setitem_indexer(self, indexer) -> bool: pass elif is_integer(i): if i >= len(ax): - raise IndexError( - f"{self.name} cannot enlarge its target object" - ) + raise IndexError("iloc cannot enlarge its target object") elif isinstance(i, dict): - raise IndexError(f"{self.name} cannot enlarge its target object") + raise IndexError("iloc cannot enlarge its target object") return True @@ -1548,8 +1546,8 @@ def _convert_to_indexer(self, key, axis: int, is_setter: bool = False): return key elif is_float(key): + # _validate_indexer call will always raise labels._validate_indexer("positional", key, "iloc") - return key self._validate_key(key, axis) return key From 4081bbd7564aed8388d264b3b16104bc5945ada0 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 18 Feb 2020 11:56:31 -0800 Subject: [PATCH 5/6] FIXUP: stray apostrophe --- pandas/core/indexing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/indexing.py b/pandas/core/indexing.py index 2d00578eda8c3..391ea0e610931 100755 --- a/pandas/core/indexing.py +++ b/pandas/core/indexing.py @@ -760,7 +760,7 @@ def _getitem_lowerdim(self, tup: Tuple): # slice returns a new object. if com.is_null_slice(new_key): return section - # This is an elided recursive call to iloc/loc' + # This is an elided recursive call to iloc/loc return getattr(section, self.name)[new_key] raise IndexingError("not applicable") From eabf21165845b9b3bd79d058e852587bcd544553 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Sat, 22 Feb 2020 09:32:05 -0800 Subject: [PATCH 6/6] remove unnecessary check --- pandas/core/indexing.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/pandas/core/indexing.py b/pandas/core/indexing.py index 1ba33d3618b28..a614c731df879 100755 --- a/pandas/core/indexing.py +++ b/pandas/core/indexing.py @@ -737,13 +737,10 @@ def _getitem_lowerdim(self, tup: Tuple): # caught by the _is_nested_tuple_indexer check above. section = self._getitem_axis(key, axis=i) - if not hasattr(section, "ndim"): - # We should never get here, because _getitem_lowerdim - # is only called after a check for is_scalar_access, - # which this would be. - return section - - elif section.ndim == self.ndim: + # We should never have a scalar section here, because + # _getitem_lowerdim is only called after a check for + # is_scalar_access, which that would be. + if section.ndim == self.ndim: # we're in the middle of slicing through a MultiIndex # revise the key wrt to `section` by inserting an _NS new_key = tup[:i] + (_NS,) + tup[i + 1 :]