Skip to content

BUG: DataFrame.first_valid_index() fails if there is no valid entry. #17488

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Sep 23, 2017
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions doc/source/whatsnew/v0.21.0.txt
Original file line number Diff line number Diff line change
Expand Up @@ -399,6 +399,7 @@ Indexing
- Bug in ``CategoricalIndex`` reindexing in which specified indices containing duplicates were not being respected (:issue:`17323`)
- Bug in intersection of ``RangeIndex`` with negative step (:issue:`17296`)
- Bug in ``IntervalIndex`` where performing a scalar lookup fails for included right endpoints of non-overlapping monotonic decreasing indexes (:issue:`16417`, :issue:`17271`)
- Bug in :meth:`DataFrame.first_valid_index` and :meth:`DataFrame.last_valid_index` when no valid entry (:issue:`17400`)

I/O
^^^
Expand Down
20 changes: 12 additions & 8 deletions pandas/core/frame.py
Original file line number Diff line number Diff line change
Expand Up @@ -4069,23 +4069,27 @@ def update(self, other, join='left', overwrite=True, filter_func=None,
# ----------------------------------------------------------------------
# Misc methods

def _get_valid_indices(self):
is_valid = self.count(1) > 0
return self.index[is_valid]

@Appender(_shared_docs['valid_index'] % {
'position': 'first', 'klass': 'DataFrame'})
def first_valid_index(self):
"""
Return label for first non-NA/null value
"""
if len(self) == 0:
return None

return self.index[self.count(1) > 0][0]
valid_indices = self._get_valid_indices()
return valid_indices[0] if len(valid_indices) else None

@Appender(_shared_docs['valid_index'] % {
'position': 'first', 'klass': 'DataFrame'})
def last_valid_index(self):
"""
Return label for last non-NA/null value
"""
if len(self) == 0:
return None

return self.index[self.count(1) > 0][-1]
valid_indices = self._get_valid_indices()
return valid_indices[-1] if len(valid_indices) else None

# ----------------------------------------------------------------------
# Data reshaping
Expand Down
16 changes: 16 additions & 0 deletions pandas/core/generic.py
Original file line number Diff line number Diff line change
Expand Up @@ -6763,6 +6763,22 @@ def transform(self, func, *args, **kwargs):

cls.transform = transform

# ----------------------------------------------------------------------
# Misc methods

_shared_docs['valid_index'] = """
Return index for %(position)s non-NA/null value.

Notes
--------
If all elements are non-NA/null, returns None.
Also returns None for empty %(klass)s.

Returns
--------
scalar : type of index
"""


def _doc_parms(cls):
"""Return a tuple of the doc parms."""
Expand Down
10 changes: 4 additions & 6 deletions pandas/core/series.py
Original file line number Diff line number Diff line change
Expand Up @@ -2838,10 +2838,9 @@ def dropna(self, axis=0, inplace=False, **kwargs):
valid = lambda self, inplace=False, **kwargs: self.dropna(inplace=inplace,
**kwargs)

@Appender(generic._shared_docs['valid_index'] % {
'position': 'first', 'klass': 'Series'})
def first_valid_index(self):
"""
Return label for first non-NA/null value
"""
if len(self) == 0:
return None

Expand All @@ -2852,10 +2851,9 @@ def first_valid_index(self):
else:
return self.index[i]

@Appender(generic._shared_docs['valid_index'] % {
'position': 'last', 'klass': 'Series'})
def last_valid_index(self):
"""
Return label for last non-NA/null value
"""
if len(self) == 0:
return None

Expand Down
5 changes: 5 additions & 0 deletions pandas/tests/frame/test_timeseries.py
Original file line number Diff line number Diff line change
Expand Up @@ -440,6 +440,11 @@ def test_first_last_valid(self):
assert empty.last_valid_index() is None
assert empty.first_valid_index() is None

# GH17400: no valid entries
frame[:] = nan
assert frame.last_valid_index() is None
assert frame.first_valid_index() is None

def test_at_time_frame(self):
rng = date_range('1/1/2000', '1/5/2000', freq='5min')
ts = DataFrame(np.random.randn(len(rng), 2), index=rng)
Expand Down