optimized a little bit for speed

Chang She · wesm · commit 030ab4db6785 · 2012-05-19T13:11:39.000-04:00
diff --git a/pandas/core/frame.py b/pandas/core/frame.py
@@ -2344,7 +2344,7 @@ def dropna(self, axis=0, how='any', thresh=None, subset=None):
         new_labels = labels[mask]
         return self.reindex(**{axis_name: new_labels})
 
-    def drop_duplicates(self, cols=None, take_last=False, skipna=True):
+    def drop_duplicates(self, cols=None, take_last=False):
         """
         Return DataFrame with duplicate rows removed, optionally only
         considering certain columns
@@ -2363,10 +2363,10 @@ def drop_duplicates(self, cols=None, take_last=False, skipna=True):
         -------
         deduplicated : DataFrame
         """
-        duplicated = self.duplicated(cols, take_last=take_last, skipna=skipna)
+        duplicated = self.duplicated(cols, take_last=take_last)
         return self[-duplicated]
 
-    def duplicated(self, cols=None, take_last=False, skipna=True):
+    def duplicated(self, cols=None, take_last=False):
         """
         Return boolean Series denoting duplicate rows, optionally only
         considering certain columns
@@ -2378,29 +2378,22 @@ def duplicated(self, cols=None, take_last=False, skipna=True):
             default use all of the columns
         take_last : boolean, default False
             Take the last observed row in a row. Defaults to the first row
-        skipna : boolean, default True
-            If True then NaN are not marked as duplicates
 
         Returns
         -------
         duplicated : Series
         """
-        zip_func = lib.fast_zip if skipna else lib.fast_zip_fillna
-
         if cols is not None:
             if isinstance(cols, list):
                 values = [self[x].values for x in cols]
-                keys = zip_func(values)
-                dup_func = lib.duplicated_skipna
+                keys = lib.fast_zip_fillna(values)
             else:
-                keys = self[cols]
-                dup_func = lib.duplicated_skipna if skipna else lib.duplicated
+                keys = lib.fast_zip_fillna([self[cols]])
         else:
             values = list(self.values.T)
-            keys = zip_func(values)
-            dup_func = lib.duplicated_skipna
+            keys = lib.fast_zip_fillna(values)
 
-        duplicated = dup_func(list(keys), take_last=take_last)
+        duplicated = lib.duplicated(keys, take_last=take_last)
         return Series(duplicated, index=self.index)
 
     #----------------------------------------------------------------------
diff --git a/pandas/src/groupby.pyx b/pandas/src/groupby.pyx
@@ -1301,39 +1301,72 @@ def count_level_2d(ndarray[uint8_t, ndim=2, cast=True] mask,
 
     return counts
 
-def duplicated_skipna(list values, take_last=False):
+cdef class _PandasNull:
+
+    def __richcmp__(_PandasNull self, object other, int op):
+        if op == 2: # ==
+            return isinstance(other, _PandasNull)
+        elif op == 3: # !=
+            return not isinstance(other, _PandasNull)
+        else:
+            return False
+
+    def __hash__(self):
+        return 0
+
+pandas_null = _PandasNull()
+
+def fast_zip_fillna(list ndarrays, fill_value=pandas_null):
+    '''
+    For zipping multiple ndarrays into an ndarray of tuples
+    '''
     cdef:
-        Py_ssize_t i, n
-        dict seen = {}
-        object row
+        Py_ssize_t i, j, k, n
+        ndarray[object] result
+        flatiter it
+        object val, tup
 
-    n = len(values)
-    cdef ndarray[uint8_t] result = np.zeros(n, dtype=np.uint8)
+    k = len(ndarrays)
+    n = len(ndarrays[0])
 
-    if take_last:
-        for i from n > i >= 0:
-            row = values[i]
-            if row in seen:
-                result[i] = 1
-            else:
-                seen[row] = None
-                result[i] = 0
-    else:
-        for i from 0 <= i < n:
-            row = values[i]
-            if row in seen:
-                result[i] = 1
-            else:
-                seen[row] = None
-                result[i] = 0
+    result = np.empty(n, dtype=object)
 
-    return result.view(np.bool_)
+    # initialize tuples on first pass
+    arr = ndarrays[0]
+    it = <flatiter> PyArray_IterNew(arr)
+    for i in range(n):
+        val = PyArray_GETITEM(arr, PyArray_ITER_DATA(it))
+        tup = PyTuple_New(k)
+
+        if val != val:
+            val = fill_value
 
-def duplicated(list values, take_last=False):
+        PyTuple_SET_ITEM(tup, 0, val)
+        Py_INCREF(val)
+        result[i] = tup
+        PyArray_ITER_NEXT(it)
+
+    for j in range(1, k):
+        arr = ndarrays[j]
+        it = <flatiter> PyArray_IterNew(arr)
+        if len(arr) != n:
+            raise ValueError('all arrays must be same length')
+
+        for i in range(n):
+            val = PyArray_GETITEM(arr, PyArray_ITER_DATA(it))
+            if val != val:
+                val = fill_value
+
+            PyTuple_SET_ITEM(result[i], j, val)
+            Py_INCREF(val)
+            PyArray_ITER_NEXT(it)
+
+    return result
+
+def duplicated(ndarray[object] values, take_last=False):
     cdef:
         Py_ssize_t i, n
         dict seen = {}
-        bint has_nan = 0
         object row
 
     n = len(values)
@@ -1342,14 +1375,9 @@ def duplicated(list values, take_last=False):
     if take_last:
         for i from n > i >= 0:
             row = values[i]
+
             if row in seen:
                 result[i] = 1
-            elif row != row:
-                if has_nan:
-                    result[i] = 1
-                else:
-                    has_nan = 1
-                    result[i] = 0
             else:
                 seen[row] = None
                 result[i] = 0
@@ -1358,12 +1386,6 @@ def duplicated(list values, take_last=False):
             row = values[i]
             if row in seen:
                 result[i] = 1
-            elif row != row:
-                if has_nan:
-                    result[i] = 1
-                else:
-                    has_nan = 1
-                    result[i] = 0
             else:
                 seen[row] = None
                 result[i] = 0
diff --git a/pandas/src/tseries.pyx b/pandas/src/tseries.pyx
@@ -404,58 +404,6 @@ def fast_zip(list ndarrays):
 
     return result
 
-cdef class _PandasNull:
-    pass
-
-pandas_null = _PandasNull()
-
-def fast_zip_fillna(list ndarrays, fill_value=pandas_null):
-    '''
-    For zipping multiple ndarrays into an ndarray of tuples
-    '''
-    cdef:
-        Py_ssize_t i, j, k, n
-        ndarray[object] result
-        flatiter it
-        object val, tup
-
-    k = len(ndarrays)
-    n = len(ndarrays[0])
-
-    result = np.empty(n, dtype=object)
-
-    # initialize tuples on first pass
-    arr = ndarrays[0]
-    it = <flatiter> PyArray_IterNew(arr)
-    for i in range(n):
-        val = PyArray_GETITEM(arr, PyArray_ITER_DATA(it))
-        tup = PyTuple_New(k)
-
-        if val != val:
-            val = fill_value
-
-        PyTuple_SET_ITEM(tup, 0, val)
-        Py_INCREF(val)
-        result[i] = tup
-        PyArray_ITER_NEXT(it)
-
-    for j in range(1, k):
-        arr = ndarrays[j]
-        it = <flatiter> PyArray_IterNew(arr)
-        if len(arr) != n:
-            raise ValueError('all arrays must be same length')
-
-        for i in range(n):
-            val = PyArray_GETITEM(arr, PyArray_ITER_DATA(it))
-            if val != val:
-                val = fill_value
-
-            PyTuple_SET_ITEM(result[i], j, val)
-            Py_INCREF(val)
-            PyArray_ITER_NEXT(it)
-
-    return result
-
 def get_reverse_indexer(ndarray[int64_t] indexer, Py_ssize_t length):
     cdef:
         Py_ssize_t i, n = len(indexer)
diff --git a/pandas/tests/test_frame.py b/pandas/tests/test_frame.py
@@ -3323,20 +3323,20 @@ def test_drop_duplicates_NA(self):
                         'D' : range(8)})
 
         # single column
-        result = df.drop_duplicates('C', skipna=False)
+        result = df.drop_duplicates('C')
         expected = df[:2]
         assert_frame_equal(result, expected)
 
-        result = df.drop_duplicates('C', take_last=True, skipna=False)
+        result = df.drop_duplicates('C', take_last=True)
         expected = df.ix[[3, 7]]
         assert_frame_equal(result, expected)
 
         # multi column
-        result = df.drop_duplicates(['C', 'B'], skipna=False)
+        result = df.drop_duplicates(['C', 'B'])
         expected = df.ix[[0, 1, 2, 4]]
         assert_frame_equal(result, expected)
 
-        result = df.drop_duplicates(['C', 'B'], take_last=True, skipna=False)
+        result = df.drop_duplicates(['C', 'B'], take_last=True)
         expected = df.ix[[1, 3, 6, 7]]
         assert_frame_equal(result, expected)
 
diff --git a/pandas/tests/test_tseries.py b/pandas/tests/test_tseries.py
@@ -170,7 +170,7 @@ def test_ensure_platform_int():
     assert(result is arr)
 
 def test_duplicated_with_nas():
-    keys = [0, 1, nan, 0, 2, nan]
+    keys = np.array([0, 1, nan, 0, 2, nan], dtype=object)
 
     result = lib.duplicated(keys)
     expected = [False, False, False, True, False, True]
@@ -180,7 +180,9 @@ def test_duplicated_with_nas():
     expected = [True, False, True, False, False, False]
     assert(np.array_equal(result, expected))
 
-    keys = [(0, 0), (0, nan), (nan, 0), (nan, nan)] * 2
+    keys = np.empty(8, dtype=object)
+    for i, t in enumerate(zip([0, 0, nan, nan]*2, [0, nan, 0, nan]*2)):
+        keys[i] = t
 
     result = lib.duplicated(keys)
     falses = [False] * 4
diff --git a/vb_suite/reindex.py b/vb_suite/reindex.py
@@ -114,6 +114,7 @@ def backfill():
 
 # pathological, but realistic
 setup = common_setup + """
+import pandas._tseries as lib
 N = 10000
 K = 10
 
@@ -135,11 +136,22 @@ def backfill():
                                   name='frame_drop_duplicates',
                                   start_date=datetime(2011, 11, 15))
 
-statement2 = "df.drop_duplicates(['key1', 'key2'], skipna=False)"
-frame_drop_duplicates_na = Benchmark(statement, setup,
-                                     name='frame_drop_duplicates',
+lib_fast_zip = Benchmark('lib.fast_zip(df.values.T)', setup,
+                         name='lib_fast_zip',
+                         start_date=datetime(2012, 1, 1))
+
+setup = setup + """
+df.ix[:10000, :] = np.nan
+"""
+statement2 = "df.drop_duplicates(['key1', 'key2'])"
+frame_drop_duplicates_na = Benchmark(statement2, setup,
+                                     name='frame_drop_duplicates_na',
                                      start_date=datetime(2012, 5, 15))
 
+lib_fast_zip_fillna = Benchmark('lib.fast_zip_fillna(df.values.T)', setup,
+                                name='lib_fast_zip_fillna',
+                                start_date=datetime(2012, 5, 15))
+
 #----------------------------------------------------------------------
 # fillna, many columns
 
