Skip to content

Commit 0e6ea02

Browse files
committed
BUG: _nsorted incorrect with duplicated values in index (#13412)
Add note to whatsnew Add nlargest benchmark Add tests for Series organize nsorted methods pep 8 fixes passed test and pep8 add docstrings
1 parent 22d982a commit 0e6ea02

File tree

7 files changed

+81
-16
lines changed

7 files changed

+81
-16
lines changed

asv_bench/benchmarks/frame_methods.py

+11
Original file line numberDiff line numberDiff line change
@@ -1012,3 +1012,14 @@ def setup(self):
10121012

10131013
def time_frame_quantile_axis1(self):
10141014
self.df.quantile([0.1, 0.5], axis=1)
1015+
1016+
1017+
class frame_nlargest(object):
1018+
goal_time = 0.2
1019+
1020+
def setup(self):
1021+
self.df = DataFrame(np.random.randn(1000, 3),
1022+
columns=list('ABC'))
1023+
1024+
def time_frame_nlargest(self):
1025+
self.df.nlargest(100, 'A')

doc/source/whatsnew/v0.19.2.txt

+1-1
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ Bug Fixes
5252
- Bug in resampling a ``DatetimeIndex`` in local TZ, covering a DST change, which would raise ``AmbiguousTimeError`` (:issue:`14682`)
5353

5454

55-
55+
- Bug in ``DataFrame.nlargest`` and ``DataFrame.nsmallest`` when the index had duplicate values (:issue:`13412`)
5656

5757

5858

pandas/core/algorithms.py

+28-2
Original file line numberDiff line numberDiff line change
@@ -684,11 +684,12 @@ def select_n_slow(dropped, n, keep, method):
684684
_select_methods = {'nsmallest': nsmallest, 'nlargest': nlargest}
685685

686686

687-
def select_n(series, n, keep, method):
688-
"""Implement n largest/smallest.
687+
def select_n_series(series, n, keep, method):
688+
"""Implement n largest/smallest for pandas Series
689689
690690
Parameters
691691
----------
692+
series : pandas.Series object
692693
n : int
693694
keep : {'first', 'last'}, default 'first'
694695
method : str, {'nlargest', 'nsmallest'}
@@ -717,6 +718,31 @@ def select_n(series, n, keep, method):
717718
return dropped.iloc[inds]
718719

719720

721+
def select_n_frame(frame, columns, n, method, keep):
722+
"""Implement n largest/smallest for pandas DataFrame
723+
724+
Parameters
725+
----------
726+
series : pandas.DataFrame object
727+
columns : list or str
728+
n : int
729+
keep : {'first', 'last'}, default 'first'
730+
method : str, {'nlargest', 'nsmallest'}
731+
732+
Returns
733+
-------
734+
nordered : DataFrame
735+
"""
736+
from pandas.core.series import Series
737+
if not is_list_like(columns):
738+
columns = [columns]
739+
columns = list(columns)
740+
ser = getattr(frame[columns[0]], method)(n, keep=keep)
741+
if isinstance(ser, Series):
742+
ser = ser.to_frame()
743+
return ser.merge(frame, on=columns[0], left_index=True)[frame.columns]
744+
745+
720746
def _finalize_nsmallest(arr, kth_val, n, keep, narr):
721747
ns, = np.nonzero(arr <= kth_val)
722748
inds = ns[arr[ns].argsort(kind='mergesort')][:n]

pandas/core/frame.py

+2-11
Original file line numberDiff line numberDiff line change
@@ -3390,15 +3390,6 @@ def sortlevel(self, level=0, axis=0, ascending=True, inplace=False,
33903390
return self.sort_index(level=level, axis=axis, ascending=ascending,
33913391
inplace=inplace, sort_remaining=sort_remaining)
33923392

3393-
def _nsorted(self, columns, n, method, keep):
3394-
if not is_list_like(columns):
3395-
columns = [columns]
3396-
columns = list(columns)
3397-
ser = getattr(self[columns[0]], method)(n, keep=keep)
3398-
ascending = dict(nlargest=False, nsmallest=True)[method]
3399-
return self.loc[ser.index].sort_values(columns, ascending=ascending,
3400-
kind='mergesort')
3401-
34023393
def nlargest(self, n, columns, keep='first'):
34033394
"""Get the rows of a DataFrame sorted by the `n` largest
34043395
values of `columns`.
@@ -3431,7 +3422,7 @@ def nlargest(self, n, columns, keep='first'):
34313422
1 10 b 2
34323423
2 8 d NaN
34333424
"""
3434-
return self._nsorted(columns, n, 'nlargest', keep)
3425+
return algos.select_n_frame(self, columns, n, 'nlargest', keep)
34353426

34363427
def nsmallest(self, n, columns, keep='first'):
34373428
"""Get the rows of a DataFrame sorted by the `n` smallest
@@ -3465,7 +3456,7 @@ def nsmallest(self, n, columns, keep='first'):
34653456
0 1 a 1
34663457
2 8 d NaN
34673458
"""
3468-
return self._nsorted(columns, n, 'nsmallest', keep)
3459+
return algos.select_n_frame(self, columns, n, 'nsmallest', keep)
34693460

34703461
def swaplevel(self, i=-2, j=-1, axis=0):
34713462
"""

pandas/core/series.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -1935,7 +1935,7 @@ def nlargest(self, n=5, keep='first'):
19351935
>>> s = pd.Series(np.random.randn(1e6))
19361936
>>> s.nlargest(10) # only sorts up to the N requested
19371937
"""
1938-
return algos.select_n(self, n=n, keep=keep, method='nlargest')
1938+
return algos.select_n_series(self, n=n, keep=keep, method='nlargest')
19391939

19401940
@deprecate_kwarg('take_last', 'keep', mapping={True: 'last',
19411941
False: 'first'})
@@ -1973,7 +1973,7 @@ def nsmallest(self, n=5, keep='first'):
19731973
>>> s = pd.Series(np.random.randn(1e6))
19741974
>>> s.nsmallest(10) # only sorts up to the N requested
19751975
"""
1976-
return algos.select_n(self, n=n, keep=keep, method='nsmallest')
1976+
return algos.select_n_series(self, n=n, keep=keep, method='nsmallest')
19771977

19781978
def sortlevel(self, level=0, ascending=True, sort_remaining=True):
19791979
"""

pandas/tests/frame/test_analytics.py

+28
Original file line numberDiff line numberDiff line change
@@ -1323,6 +1323,34 @@ def test_nsmallest_multiple_columns(self):
13231323
expected = df.sort_values(['a', 'c']).head(5)
13241324
tm.assert_frame_equal(result, expected)
13251325

1326+
def test_nsmallest_nlargest_duplicate_index(self):
1327+
df = pd.DataFrame({'a': [1, 2, 3, 4],
1328+
'b': [4, 3, 2, 1],
1329+
'c': [0, 1, 2, 3]},
1330+
index=[0, 0, 1, 1])
1331+
result = df.nsmallest(4, 'a')
1332+
expected = df.sort_values('a').head(4)
1333+
tm.assert_frame_equal(result, expected)
1334+
1335+
result = df.nlargest(4, 'a')
1336+
expected = df.sort_values('a', ascending=False).head(4)
1337+
tm.assert_frame_equal(result, expected)
1338+
1339+
result = df.nsmallest(4, ['a', 'c'])
1340+
expected = df.sort_values(['a', 'c']).head(4)
1341+
tm.assert_frame_equal(result, expected)
1342+
1343+
result = df.nsmallest(4, ['c', 'a'])
1344+
expected = df.sort_values(['c', 'a']).head(4)
1345+
tm.assert_frame_equal(result, expected)
1346+
1347+
result = df.nlargest(4, ['a', 'c'])
1348+
expected = df.sort_values(['a', 'c'], ascending=False).head(4)
1349+
tm.assert_frame_equal(result, expected)
1350+
1351+
result = df.nlargest(4, ['c', 'a'])
1352+
expected = df.sort_values(['c', 'a'], ascending=False).head(4)
1353+
tm.assert_frame_equal(result, expected)
13261354
# ----------------------------------------------------------------------
13271355
# Isin
13281356

pandas/tests/series/test_analytics.py

+9
Original file line numberDiff line numberDiff line change
@@ -1515,6 +1515,15 @@ def test_nsmallest_nlargest(self):
15151515
with tm.assertRaisesRegexp(ValueError, msg):
15161516
s.nlargest(keep='invalid')
15171517

1518+
# GH 13412
1519+
s = Series([1, 4, 3, 2], index=[0, 0, 1, 1])
1520+
result = s.nlargest(3)
1521+
expected = s.sort_values(ascending=False).head(3)
1522+
assert_series_equal(result, expected)
1523+
result = s.nsmallest(3)
1524+
expected = s.sort_values().head(3)
1525+
assert_series_equal(result, expected)
1526+
15181527
def test_sortlevel(self):
15191528
mi = MultiIndex.from_tuples([[1, 1, 3], [1, 1, 1]], names=list('ABC'))
15201529
s = Series([1, 2], mi)

0 commit comments

Comments
 (0)