Merge pull request pandas-dev#11159 from sinhrks/dropna_perf

PERF: Series.dropna with non-nan dtype blocks

Author: jreback

asv_bench/benchmarks/series_methods.py

@@ -71,3 +71,23 @@ def setup(self):
     def time_series_nsmallest2(self):
         self.s2.nsmallest(3, take_last=True)
         self.s2.nsmallest(3, take_last=False)
+
+
+class series_dropna_int64(object):
+    goal_time = 0.2
+
+    def setup(self):
+        self.s = Series(np.random.randint(1, 10, 1000000))
+
+    def time_series_dropna_int64(self):
+        self.s.dropna()
+
+class series_dropna_datetime(object):
+    goal_time = 0.2
+
+    def setup(self):
+        self.s = Series(pd.date_range('2000-01-01', freq='S', periods=1000000))
+        self.s[np.random.randint(1, 1000000, 100)] = pd.NaT
+
+    def time_series_dropna_datetime(self):
+        self.s.dropna()

doc/source/whatsnew/v0.17.1.txt

@@ -57,6 +57,7 @@ Performance Improvements
 ~~~~~~~~~~~~~~~~~~~~~~~~
 
 - Checking monotonic-ness before sorting on an index (:issue:`11080`)
+- ``Series.dropna`` performance improvement when its dtype can't contain ``NaN`` (:issue:`11159`)
 
 
 - Release the GIL on most datetime field operations (e.g. ``DatetimeIndex.year``, ``Series.dt.year``), normalization, and conversion to and from ``Period``, ``DatetimeIndex.to_period`` and ``PeriodIndex.to_timestamp`` (:issue:`11263`)

pandas/core/series.py

@@ -2501,11 +2501,19 @@ def dropna(self, axis=0, inplace=False, **kwargs):
                     'argument "{0}"'.format(list(kwargs.keys())[0]))
 
         axis = self._get_axis_number(axis or 0)
-        result = remove_na(self)
-        if inplace:
-            self._update_inplace(result)
+
+        if self._can_hold_na:
+            result = remove_na(self)
+            if inplace:
+                self._update_inplace(result)
+            else:
+                return result
         else:
-            return result
+            if inplace:
+                # do nothing
+                pass
+            else:
+                return self.copy()
 
     valid = lambda self, inplace=False, **kwargs: self.dropna(inplace=inplace,
                                                               **kwargs)

pandas/tests/test_series.py

@@ -5169,7 +5169,6 @@ def test_dropna_empty(self):
         # invalid axis
         self.assertRaises(ValueError, s.dropna, axis=1)
 
-
     def test_datetime64_tz_dropna(self):
         # DatetimeBlock
         s = Series([Timestamp('2011-01-01 10:00'), pd.NaT,
@@ -5192,6 +5191,18 @@ def test_datetime64_tz_dropna(self):
         self.assertEqual(result.dtype, 'datetime64[ns, Asia/Tokyo]')
         self.assert_series_equal(result, expected)
 
+    def test_dropna_no_nan(self):
+        for s in [Series([1, 2, 3], name='x'),
+                  Series([False, True, False], name='x')]:
+
+            result = s.dropna()
+            self.assert_series_equal(result, s)
+            self.assertFalse(result is s)
+
+            s2 = s.copy()
+            s2.dropna(inplace=True)
+            self.assert_series_equal(s2, s)
+
     def test_axis_alias(self):
         s = Series([1, 2, np.nan])
         assert_series_equal(s.dropna(axis='rows'), s.dropna(axis='index'))