Skip to content

ENH: support Ellipsis in loc/iloc #37750

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 26 commits into from
Oct 16, 2021
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
9573948
ENH: support Ellipsis in loc/iloc
jbrockmendel Nov 10, 2020
3033365
mypy fixup
jbrockmendel Nov 11, 2020
10e8048
Merge branch 'master' of https://github.com/pandas-dev/pandas into en…
jbrockmendel Nov 11, 2020
9ed4838
Merge branch 'master' of https://github.com/pandas-dev/pandas into en…
jbrockmendel Nov 24, 2020
7827c8d
Merge branch 'master' of https://github.com/pandas-dev/pandas into en…
jbrockmendel Dec 24, 2020
d0888df
Merge branch 'master' of https://github.com/pandas-dev/pandas into en…
jbrockmendel Jan 12, 2021
a703e49
Merge branch 'master' of https://github.com/pandas-dev/pandas into en…
jbrockmendel Jan 18, 2021
d109346
Merge branch 'master' of https://github.com/pandas-dev/pandas into en…
jbrockmendel Jan 19, 2021
9445cf4
rename per request
jbrockmendel Jan 19, 2021
6d33e00
Merge branch 'master' of https://github.com/pandas-dev/pandas into en…
jbrockmendel Jan 20, 2021
a53f9a2
TST: flesh out tests
jbrockmendel Jan 20, 2021
843a5da
Merge branch 'master' into enh-indexing-ellipses
jbrockmendel Feb 2, 2021
fbec549
Merge branch 'master' into enh-indexing-ellipses
jbrockmendel Feb 3, 2021
8b637ee
Merge branch 'master' of https://github.com/pandas-dev/pandas into en…
jbrockmendel Feb 4, 2021
ac76a33
Merge branch 'master' into enh-indexing-ellipses
jbrockmendel Feb 16, 2021
351b65b
Merge branch 'master' into enh-indexing-ellipses
jbrockmendel Feb 17, 2021
3797b42
Merge branch 'master' into enh-indexing-ellipses
jbrockmendel Mar 4, 2021
59a0e58
Merge branch 'master' into enh-indexing-ellipses
jbrockmendel Mar 6, 2021
8c656a2
Merge branch 'master' into enh-indexing-ellipses
jbrockmendel Apr 8, 2021
a4e9fc7
Merge branch 'master' into enh-indexing-ellipses
jbrockmendel Jun 1, 2021
a5632f3
whatsnew
jbrockmendel Jun 1, 2021
1953d49
Merge branch 'master' into enh-indexing-ellipses
jbrockmendel Jun 3, 2021
8515d04
Merge branch 'master' into enh-indexing-ellipses
jbrockmendel Sep 28, 2021
c41d0fc
Merge branch 'master' into enh-indexing-ellipses
jbrockmendel Oct 5, 2021
a2eb766
Merge branch 'master' into enh-indexing-ellipses
jbrockmendel Oct 6, 2021
d186fa3
Merge branch 'master' into enh-indexing-ellipses
jbrockmendel Oct 16, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 38 additions & 7 deletions pandas/core/indexing.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
from contextlib import suppress
from typing import TYPE_CHECKING, Any, Hashable, List, Sequence, Tuple, Union
from typing import TYPE_CHECKING, Hashable, List, Tuple, Union
import warnings

import numpy as np
Expand Down Expand Up @@ -705,11 +705,31 @@ def _validate_key(self, key, axis: int):
"""
raise AbstractMethodError(self)

def _expand_ellipsis(self, tup: Tuple) -> Tuple:
"""
If a tuple key includes an Ellipsis, replace it with an appropriate
number of null slices.
"""
if any(x is Ellipsis for x in tup):

if len(tup) == self.ndim:
# It is unambiguous what axis this Ellipsis is indexing,
# treat as a single null slice.
i = tup.index(Ellipsis)
# FIXME: this assumes only one Ellipsis
new_key = tup[:i] + (_NS,) + tup[i + 1 :]
return new_key

# TODO: other cases? only one test gets here, and that is covered
# by _validate_key_length
return tup

def _has_valid_tuple(self, key: Tuple):
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you rename this method if it now actually returns a modified key?

"""
Check the key for valid keys across my indexer.
"""
self._validate_key_length(key)
key = self._validate_key_length(key)
key = self._expand_ellipsis(key)
for i, k in enumerate(key):
try:
self._validate_key(k, i)
Expand All @@ -718,6 +738,7 @@ def _has_valid_tuple(self, key: Tuple):
"Location based indexing can only have "
f"[{self._valid_types}] types"
) from err
return key

def _is_nested_tuple_indexer(self, tup: Tuple) -> bool:
"""
Expand Down Expand Up @@ -748,9 +769,14 @@ def _convert_tuple(self, key, is_setter: bool = False):

return tuple(keyidx)

def _validate_key_length(self, key: Sequence[Any]) -> None:
def _validate_key_length(self, key: Tuple) -> Tuple:
if len(key) > self.ndim:
if key[0] is Ellipsis:
# e.g. Series.iloc[..., 3] reduces to just Series.iloc[3]
key = key[1:]
return self._validate_key_length(key)
raise IndexingError("Too many indexers")
return key

def _getitem_tuple_same_dim(self, tup: Tuple):
"""
Expand Down Expand Up @@ -790,7 +816,7 @@ def _getitem_lowerdim(self, tup: Tuple):
with suppress(IndexingError):
return self._handle_lowerdim_multi_index_axis0(tup)

self._validate_key_length(tup)
tup = self._validate_key_length(tup)

for i, key in enumerate(tup):
if is_label_like(key):
Expand Down Expand Up @@ -1049,10 +1075,11 @@ def _getitem_iterable(self, key, axis: int):

def _getitem_tuple(self, tup: Tuple):
with suppress(IndexingError):
tup = self._expand_ellipsis(tup)
return self._getitem_lowerdim(tup)

# no multi-index, so validate all of the indexers
self._has_valid_tuple(tup)
tup = self._has_valid_tuple(tup)

# ugly hack for GH #836
if self._multi_take_opportunity(tup):
Expand Down Expand Up @@ -1447,7 +1474,7 @@ def _validate_integer(self, key: int, axis: int) -> None:

def _getitem_tuple(self, tup: Tuple):

self._has_valid_tuple(tup)
tup = self._has_valid_tuple(tup)
with suppress(IndexingError):
return self._getitem_lowerdim(tup)

Expand Down Expand Up @@ -2329,7 +2356,11 @@ def is_label_like(key) -> bool:
bool
"""
# select a label or row
return not isinstance(key, slice) and not is_list_like_indexer(key)
return (
not isinstance(key, slice)
and not is_list_like_indexer(key)
and key is not Ellipsis
)


def need_slice(obj) -> bool:
Expand Down
21 changes: 21 additions & 0 deletions pandas/tests/indexing/test_loc.py
Original file line number Diff line number Diff line change
Expand Up @@ -1126,6 +1126,27 @@ def test_loc_setitem_listlike_with_timedelta64index(self, indexer, expected):

tm.assert_frame_equal(expected, df)

def test_loc_getitem_ellipses_series(self):
ser = Series(range(5))

result = ser.loc[..., [3]]
expected = ser.loc[[3]]
tm.assert_series_equal(result, expected)

# also check iloc while we're here
result = ser.iloc[..., [3]]
expected = ser.iloc[[3]]
tm.assert_series_equal(result, expected)

df = ser.to_frame()
result = df.loc[..., [0]]
expected = df.loc[:, [0]]
tm.assert_frame_equal(result, expected)

result = df.iloc[..., [0]]
expected = df.iloc[:, [0]]
tm.assert_frame_equal(result, expected)


class TestLocWithMultiIndex:
@pytest.mark.parametrize(
Expand Down