BUG: fix Series.unique with 'different' NA values in an object array #714

Author: wesm

pandas/core/series.py

@@ -6,9 +6,7 @@
 # pylint: disable=W0703,W0622,W0613,W0201
 
 from itertools import izip
-import csv
 import operator
-import types
 from distutils.version import LooseVersion
 
 from numpy import nan, ndarray
@@ -815,11 +813,13 @@ def unique(self):
         """
         values = self.values
         if issubclass(values.dtype.type, np.floating):
+            if values.dtype != np.float64:
+                values = values.astype(np.float64)
             table = lib.Float64HashTable(len(values))
-            uniques = np.array(table.unique(values), dtype='f8')
+            uniques = np.array(table.unique(values), dtype=np.float64)
         else:
             if not values.dtype == np.object_:
-                values = values.astype('O')
+                values = values.astype(np.object_)
             table = lib.PyObjectHashTable(len(values))
             uniques = lib.list_to_object_array(table.unique(values))
             uniques = lib.maybe_convert_objects(uniques)

pandas/src/hashtable.pyx

@@ -728,13 +728,19 @@ cdef class PyObjectHashTable:
             object val
             khiter_t k
             list uniques = []
+            bint seen_na = 0
 
         for i in range(n):
             val = values[i]
-            k = kh_get_pymap(self.table, <PyObject*>val)
-            if k == self.table.n_buckets:
-                k = kh_put_pymap(self.table, <PyObject*>val, &ret)
-                uniques.append(val)
+
+            if not _checknull(val):
+                k = kh_get_pymap(self.table, <PyObject*>val)
+                if k == self.table.n_buckets:
+                    k = kh_put_pymap(self.table, <PyObject*>val, &ret)
+                    uniques.append(val)
+            elif not seen_na:
+                seen_na = 1
+                uniques.append(ONAN)
 
         return uniques
 

pandas/tests/test_series.py

@@ -1412,6 +1412,24 @@ def test_value_counts_nunique(self):
         expected = Series([])
         assert_series_equal(hist, expected)
 
+    def test_unique(self):
+        # 714 also, dtype=float
+        s = Series([1.2345] * 100)
+        s[::2] = np.nan
+        result = s.unique()
+        self.assert_(len(result) == 2)
+
+        s = Series([1.2345] * 100, dtype='f4')
+        s[::2] = np.nan
+        result = s.unique()
+        self.assert_(len(result) == 2)
+
+        # NAs in object arrays #714
+        s = Series(['foo'] * 100, dtype='O')
+        s[::2] = np.nan
+        result = s.unique()
+        self.assert_(len(result) == 2)
+
     def test_sort(self):
         ts = self.ts.copy()
         ts.sort()