BUG: SparseSeries.dropna should point to special function, return dense Series in case of NA fill value, GH #730

Author: wesm

pandas/core/series.py

@@ -2051,7 +2051,7 @@ def dropna(self):
         """
         return remove_na(self)
 
-    valid = dropna
+    valid = lambda self: self.dropna()
 
     isnull = isnull
     notnull = notnull

pandas/sparse/series.py

@@ -456,13 +456,16 @@ def _attach_meta(self, sparse_arr):
         sparse_series.name = self.name
         return sparse_series
 
-    def valid(self):
+    def dropna(self):
         """
-        Analogous to Series.valid
+        Analogous to Series.dropna. If fill_value=NaN, returns a dense Series
         """
         # TODO: make more efficient
         dense_valid = self.to_dense().valid()
-        return dense_valid.to_sparse(fill_value=self.fill_value)
+        if isnull(self.fill_value):
+            return dense_valid
+        else:
+            return dense_valid.to_sparse(fill_value=self.fill_value)
 
     def shift(self, periods, offset=None, timeRule=None):
         """

pandas/sparse/tests/test_sparse.py

@@ -17,7 +17,7 @@
 from pandas import Series, DataFrame, DateRange, Panel
 from pandas.core.datetools import BDay
 import pandas.core.datetools as datetools
-import pandas.util.testing as testing
+import pandas.util.testing as tm
 
 import pandas.sparse.frame as spf
 
@@ -560,7 +560,7 @@ def _compare_all(obj):
         series.fill_value = 2
         _compare_all(series)
 
-    def test_valid(self):
+    def test_dropna(self):
         sp = SparseSeries([0, 0, 0, nan, nan, 5, 6],
                           fill_value=0)
 
@@ -570,6 +570,11 @@ def test_valid(self):
         self.assert_(sp_valid.index.equals(sp.to_dense().valid().index))
         self.assertEquals(len(sp_valid.sp_values), 2)
 
+        result = self.bseries.dropna()
+        expected = self.bseries.to_dense().dropna()
+        self.assert_(not isinstance(result, SparseSeries))
+        tm.assert_series_equal(result, expected)
+
     def test_homogenize(self):
         def _check_matches(indices, expected):
             data = {}
@@ -789,7 +794,7 @@ def test_dense_to_sparse(self):
         self.assert_(isinstance(sdf, SparseDataFrame))
         self.assert_(np.isnan(sdf.default_fill_value))
         self.assert_(isinstance(sdf['A'].sp_index, BlockIndex))
-        testing.assert_frame_equal(sdf.to_dense(), df)
+        tm.assert_frame_equal(sdf.to_dense(), df)
 
         sdf = df.to_sparse(kind='integer')
         self.assert_(isinstance(sdf['A'].sp_index, IntIndex))
@@ -798,7 +803,7 @@ def test_dense_to_sparse(self):
                         'B' : [1, 2, 0, 0, 0]}, dtype=float)
         sdf = df.to_sparse(fill_value=0)
         self.assertEquals(sdf.default_fill_value, 0)
-        testing.assert_frame_equal(sdf.to_dense(), df)
+        tm.assert_frame_equal(sdf.to_dense(), df)
 
     def test_sparse_to_dense(self):
         pass