BUG: handle NAs in Series.value_counts and describe with dtype=object, GH #277

Author: wesm

pandas/core/series.py

@@ -604,15 +604,15 @@ def value_counts(self):
         """
         Returns Series containing counts of unique values. The resulting Series
         will be in descending order so that the first element is the most
-        frequently-occurring element
+        frequently-occurring element. Excludes NA values
 
         Returns
         -------
         counts : Series
         """
         from collections import defaultdict
         counter = defaultdict(lambda: 0)
-        for value in self.values:
+        for value in self.dropna().values:
             counter[value] += 1
         return Series(counter).order(ascending=False)
 
@@ -906,7 +906,7 @@ def describe(self):
         if self.dtype == object:
             names = ['count', 'unique', 'top', 'freq']
 
-            objcounts = Counter(self)
+            objcounts = Counter(self.dropna().values)
             top, freq = objcounts.most_common(1)[0]
             data = [self.count(), len(objcounts), top, freq]
 

pandas/tests/test_series.py

@@ -555,7 +555,13 @@ def test_quantile(self):
     def test_describe(self):
         _ = self.series.describe()
         _ = self.ts.describe()
-        _ = self.objSeries.describe()
+
+    def test_describe_objects(self):
+        s = Series(['a', 'b', 'b', np.nan, np.nan, np.nan, 'c', 'd', 'a', 'a'])
+        result = s.describe()
+        expected = Series({'count' : 7, 'unique' : 4,
+                           'top' : 'a', 'freq' : 3}, index=result.index)
+        assert_series_equal(result, expected)
 
     def test_append(self):
         appendedSeries = self.series.append(self.ts)
@@ -770,6 +776,12 @@ def test_value_counts(self):
         expected = Series([4, 3, 2, 1], index=['b', 'a', 'd', 'c'])
         assert_series_equal(hist, expected)
 
+        # handle NA's properly
+        s[5:7] = np.nan
+        hist = s.value_counts()
+        expected = s.dropna().value_counts()
+        assert_series_equal(hist, expected)
+
         s = Series({})
         hist = s.value_counts()
         expected = Series([])

pandas/util/testing.py

@@ -1,3 +1,5 @@
+from __future__ import division
+
 # pylint: disable-msg=W0402
 
 from datetime import datetime
@@ -116,7 +118,7 @@ def assert_dict_equal(a, b, compare_keys=True):
         assert_almost_equal(a[k], b[k])
 
 def assert_series_equal(left, right):
-    assert_almost_equal(left, right)
+    assert_almost_equal(left.values, right.values)
     assert(left.dtype == right.dtype)
     assert(left.index.equals(right.index))