Skip to content

PERF: Improve performance for array equal fast #50850

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Jan 20, 2023
Merged

PERF: Improve performance for array equal fast #50850

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
69bfe5f
PERF: Improve performance for array equal fast
phofl Jan 18, 2023
f9cab07
Merge branch 'main' into perf_array_equal_fast
phofl Jan 19, 2023
48e1635
Rename
phofl Jan 19, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions pandas/_libs/lib.pyi
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -240,6 +240,6 @@ def get_reverse_indexer(
) -> npt.NDArray[np.intp]: ...
def is_bool_list(obj: list) -> bool: ...
def dtypes_all_equal(types: list[DtypeObj]) -> bool: ...
def array_equal_fast(
left: np.ndarray, right: np.ndarray # np.ndarray[np.int64, ndim=1]
def indexer_equal_fast(
left: np.ndarray, n: int # np.ndarray[np.int64, ndim=1]
) -> bool: ...
10 changes: 4 additions & 6 deletions pandas/_libs/lib.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -650,22 +650,20 @@ ctypedef fused int6432_t:

@cython.wraparound(False)
@cython.boundscheck(False)
def array_equal_fast(
ndarray[int6432_t, ndim=1] left, ndarray[int6432_t, ndim=1] right,
) -> bool:
def indexer_equal_fast(ndarray[int6432_t, ndim=1] left, int n) -> bool:
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

While renaming this function, maybe is_range_indexer might be more appropriate? Or at least clarifying that this only compares if the indexer is equal to range(n)

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah thought about this myself and agree.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Changed

"""
Perform an element by element comparison on 1-d integer arrays, meant for indexer
comparisons
"""
cdef:
Py_ssize_t i, n = left.size
Py_ssize_t i

if left.size != right.size:
if left.size != n:
return False

for i in range(n):

if left[i] != right[i]:
if left[i] != i:
return False

return True
Expand Down
4 changes: 2 additions & 2 deletions pandas/core/frame.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@
from pandas._libs.hashtable import duplicated
from pandas._libs.lib import (
NoDefault,
array_equal_fast,
indexer_equal_fast,
no_default,
)
from pandas._typing import (
Expand Down Expand Up @@ -6724,7 +6724,7 @@ def sort_values(
else:
return self.copy(deep=None)

if array_equal_fast(indexer, np.arange(0, len(indexer), dtype=indexer.dtype)):
if indexer_equal_fast(indexer, len(indexer)):
if inplace:
return self._update_inplace(self)
else:
Expand Down
7 changes: 2 additions & 5 deletions pandas/core/generic.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@
)

from pandas._libs import lib
from pandas._libs.lib import array_equal_fast
from pandas._libs.lib import indexer_equal_fast
from pandas._libs.tslibs import (
Period,
Tick,
Expand Down Expand Up @@ -3780,10 +3780,7 @@ def _take(
axis == 0
and indices.ndim == 1
and using_copy_on_write()
and array_equal_fast(
indices,
np.arange(0, len(self), dtype=np.intp),
)
and indexer_equal_fast(indices, len(self))
):
return self.copy(deep=None)

Expand Down
8 changes: 3 additions & 5 deletions pandas/core/series.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@
reshape,
)
from pandas._libs.lib import (
array_equal_fast,
indexer_equal_fast,
no_default,
)
from pandas._typing import (
Expand Down Expand Up @@ -891,7 +891,7 @@ def take(self, indices, axis: Axis = 0, **kwargs) -> Series:
if (
indices.ndim == 1
and using_copy_on_write()
and array_equal_fast(indices, np.arange(0, len(self), dtype=indices.dtype))
and indexer_equal_fast(indices, len(self))
):
return self.copy(deep=None)

Expand Down Expand Up @@ -3566,9 +3566,7 @@ def sort_values(
values_to_sort = ensure_key_mapped(self, key)._values if key else self._values
sorted_index = nargsort(values_to_sort, kind, bool(ascending), na_position)

if array_equal_fast(
sorted_index, np.arange(0, len(sorted_index), dtype=sorted_index.dtype)
):
if indexer_equal_fast(sorted_index, len(sorted_index)):
if inplace:
return self._update_inplace(self)
return self.copy(deep=None)
Expand Down
19 changes: 8 additions & 11 deletions pandas/tests/libs/test_lib.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -244,25 +244,22 @@ def test_get_reverse_indexer(self):
tm.assert_numpy_array_equal(result, expected)

@pytest.mark.parametrize("dtype", ["int64", "int32"])
def test_array_equal_fast(self, dtype):
def test_indexer_equal_fast(self, dtype):
# GH#50592
left = np.arange(1, 100, dtype=dtype)
right = np.arange(1, 100, dtype=dtype)
assert lib.array_equal_fast(left, right)
left = np.arange(0, 100, dtype=dtype)
assert lib.indexer_equal_fast(left, 100)

@pytest.mark.parametrize("dtype", ["int64", "int32"])
def test_array_equal_fast_not_equal(self, dtype):
def test_indexer_equal_fast_not_equal(self, dtype):
# GH#50592
left = np.array([1, 2], dtype=dtype)
right = np.array([2, 2], dtype=dtype)
assert not lib.array_equal_fast(left, right)
assert not lib.indexer_equal_fast(left, 2)

@pytest.mark.parametrize("dtype", ["int64", "int32"])
def test_array_equal_fast_not_equal_shape(self, dtype):
def test_indexer_equal_fast_not_equal_shape(self, dtype):
# GH#50592
left = np.array([1, 2, 3], dtype=dtype)
right = np.array([2, 2], dtype=dtype)
assert not lib.array_equal_fast(left, right)
left = np.array([0, 1, 2], dtype=dtype)
assert not lib.indexer_equal_fast(left, 2)


def test_cache_readonly_preserve_docstrings():
Expand Down