REGR: assure .unique of mixed strings does not stringize

closes pandas-dev#16107

Author: jreback

pandas/core/algorithms.py

@@ -39,7 +39,7 @@
 # dtype access    #
 # --------------- #
 
-def _ensure_data(values, dtype=None):
+def _ensure_data(values, dtype=None, infer=True):
     """
     routine to ensure that our data is of the correct
     input dtype for lower-level routines
@@ -57,10 +57,15 @@ def _ensure_data(values, dtype=None):
     values : array-like
     dtype : pandas_dtype, optional
         coerce to this dtype
+    infer : boolean, default True
+        infer object dtypes
 
     Returns
     -------
-    (ndarray, pandas_dtype, algo dtype as a string)
+    (ndarray,
+     pandas_dtype,
+     algo dtype as a string,
+     inferred type as a string or None)
 
     """
 
@@ -69,28 +74,40 @@ def _ensure_data(values, dtype=None):
         if is_bool_dtype(values) or is_bool_dtype(dtype):
             # we are actually coercing to uint64
             # until our algos suppport uint8 directly (see TODO)
-            return np.asarray(values).astype('uint64'), 'bool', 'uint64'
+            return np.asarray(values).astype('uint64'), 'bool', 'uint64', None
         elif is_signed_integer_dtype(values) or is_signed_integer_dtype(dtype):
-            return _ensure_int64(values), 'int64', 'int64'
+            return _ensure_int64(values), 'int64', 'int64', None
         elif (is_unsigned_integer_dtype(values) or
               is_unsigned_integer_dtype(dtype)):
-            return _ensure_uint64(values), 'uint64', 'uint64'
+            return _ensure_uint64(values), 'uint64', 'uint64', None
         elif is_float_dtype(values) or is_float_dtype(dtype):
-            return _ensure_float64(values), 'float64', 'float64'
+            return _ensure_float64(values), 'float64', 'float64', None
         elif is_object_dtype(values) and dtype is None:
-            return _ensure_object(np.asarray(values)), 'object', 'object'
+
+            # if we can infer a numeric then do this
+            inferred = None
+            if infer:
+                inferred = lib.infer_dtype(values)
+                if inferred in ['integer']:
+                    return _ensure_int64(values), 'int64', 'int64', inferred
+                elif inferred in ['floating']:
+                    return (_ensure_float64(values),
+                            'float64', 'float64', inferred)
+
+            return (_ensure_object(np.asarray(values)),
+                    'object', 'object', inferred)
         elif is_complex_dtype(values) or is_complex_dtype(dtype):
 
             # ignore the fact that we are casting to float
             # which discards complex parts
             with catch_warnings(record=True):
                 values = _ensure_float64(values)
-            return values, 'float64', 'float64'
+            return values, 'float64', 'float64', None
 
-    except (TypeError, ValueError):
+    except (TypeError, ValueError, OverflowError):
         # if we are trying to coerce to a dtype
         # and it is incompat this will fall thru to here
-        return _ensure_object(values), 'object', 'object'
+        return _ensure_object(values), 'object', 'object', None
 
     # datetimelike
     if (needs_i8_conversion(values) or
@@ -111,7 +128,7 @@ def _ensure_data(values, dtype=None):
             values = DatetimeIndex(values)
             dtype = values.dtype
 
-        return values.asi8, dtype, 'int64'
+        return values.asi8, dtype, 'int64', None
 
     elif is_categorical_dtype(values) or is_categorical_dtype(dtype):
         values = getattr(values, 'values', values)
@@ -122,11 +139,11 @@ def _ensure_data(values, dtype=None):
         # until our algos suppport int* directly (not all do)
         values = _ensure_int64(values)
 
-        return values, dtype, 'int64'
+        return values, dtype, 'int64', None
 
     # we have failed, return object
     values = np.asarray(values)
-    return _ensure_object(values), 'object', 'object'
+    return _ensure_object(values), 'object', 'object', None
 
 
 def _reconstruct_data(values, dtype, original):
@@ -150,7 +167,13 @@ def _reconstruct_data(values, dtype, original):
     elif is_datetime64tz_dtype(dtype) or is_period_dtype(dtype):
         values = Index(original)._shallow_copy(values, name=None)
     elif dtype is not None:
-        values = values.astype(dtype)
+
+        # don't cast to object if we are numeric
+        if is_object_dtype(dtype):
+            if not is_numeric_dtype(values):
+                values = values.astype(dtype)
+        else:
+            values = values.astype(dtype)
 
     return values
 
@@ -161,7 +184,7 @@ def _ensure_arraylike(values):
     """
     if not isinstance(values, (np.ndarray, ABCCategorical,
                                ABCIndexClass, ABCSeries)):
-        values = np.array(values)
+        values = np.array(values, dtype=object)
     return values
 
 
@@ -174,11 +197,13 @@ def _ensure_arraylike(values):
 }
 
 
-def _get_hashtable_algo(values):
+def _get_hashtable_algo(values, infer=False):
     """
     Parameters
     ----------
     values : arraylike
+    infer : boolean, default False
+        infer object dtypes
 
     Returns
     -------
@@ -188,12 +213,14 @@ def _get_hashtable_algo(values):
            dtype,
            ndtype)
     """
-    values, dtype, ndtype = _ensure_data(values)
+    values, dtype, ndtype, inferred = _ensure_data(values, infer=infer)
 
     if ndtype == 'object':
 
         # its cheaper to use a String Hash Table than Object
-        if lib.infer_dtype(values) in ['string']:
+        if inferred is None:
+            inferred = lib.infer_dtype(values)
+        if inferred in ['string']:
             ndtype = 'string'
         else:
             ndtype = 'object'
@@ -202,24 +229,43 @@ def _get_hashtable_algo(values):
     return (htable, table, values, dtype, ndtype)
 
 
-def _get_data_algo(values, func_map):
+def _get_data_algo(values, func_map, dtype=None, infer=False):
+    """
+    Parameters
+    ----------
+    values : array-like
+    func_map : an inferred -> function dict
+    dtype : dtype, optional
+        the requested dtype
+    infer : boolean, default False
+        infer object dtypes
+
+    Returns
+    -------
+    (function,
+     values,
+     ndtype)
+    """
 
     if is_categorical_dtype(values):
         values = values._values_for_rank()
 
-    values, dtype, ndtype = _ensure_data(values)
+    values, dtype, ndtype, inferred = _ensure_data(
+        values, dtype=dtype, infer=infer)
     if ndtype == 'object':
 
         # its cheaper to use a String Hash Table than Object
-        if lib.infer_dtype(values) in ['string']:
+        if inferred is None:
+            inferred = lib.infer_dtype(values)
+        if inferred in ['string']:
             try:
                 f = func_map['string']
             except KeyError:
                 pass
 
     f = func_map.get(ndtype, func_map['object'])
 
-    return f, values
+    return f, values, ndtype
 
 
 # --------------- #
@@ -248,7 +294,7 @@ def match(to_match, values, na_sentinel=-1):
     """
     values = com._asarray_tuplesafe(values)
     htable, _, values, dtype, ndtype = _get_hashtable_algo(values)
-    to_match, _, _ = _ensure_data(to_match, dtype)
+    to_match, _, _, _ = _ensure_data(to_match, dtype)
     table = htable(min(len(to_match), 1000000))
     table.map_locations(values)
     result = table.lookup(to_match)
@@ -344,7 +390,7 @@ def unique(values):
         return values.unique()
 
     original = values
-    htable, _, values, dtype, ndtype = _get_hashtable_algo(values)
+    htable, _, values, dtype, ndtype = _get_hashtable_algo(values, infer=False)
 
     table = htable(len(values))
     uniques = table.unique(values)
@@ -389,8 +435,8 @@ def isin(comps, values):
     if not isinstance(values, (ABCIndex, ABCSeries, np.ndarray)):
         values = np.array(list(values), dtype='object')
 
-    comps, dtype, _ = _ensure_data(comps)
-    values, _, _ = _ensure_data(values, dtype=dtype)
+    comps, dtype, _, _ = _ensure_data(comps)
+    values, _, _, _ = _ensure_data(values, dtype=dtype)
 
     # GH11232
     # work-around for numpy < 1.8 and comparisions on py3
@@ -499,7 +545,7 @@ def sort_mixed(values):
 
     if sorter is None:
         # mixed types
-        (hash_klass, _), values = _get_data_algo(values, _hashtables)
+        (hash_klass, _), values, _ = _get_data_algo(values, _hashtables)
         t = hash_klass(len(values))
         t.map_locations(values)
         sorter = _ensure_platform_int(t.lookup(ordered))
@@ -545,8 +591,8 @@ def factorize(values, sort=False, order=None, na_sentinel=-1, size_hint=None):
 
     values = _ensure_arraylike(values)
     original = values
-    values, dtype, _ = _ensure_data(values)
-    (hash_klass, vec_klass), values = _get_data_algo(values, _hashtables)
+    values, dtype, _, _ = _ensure_data(values)
+    (hash_klass, vec_klass), values, _ = _get_data_algo(values, _hashtables)
 
     table = hash_klass(size_hint or len(values))
     uniques = vec_klass()
@@ -660,7 +706,7 @@ def _value_counts_arraylike(values, dropna):
     """
     values = _ensure_arraylike(values)
     original = values
-    values, dtype, ndtype = _ensure_data(values)
+    values, dtype, ndtype, inferred = _ensure_data(values)
 
     if needs_i8_conversion(dtype):
         # i8
@@ -711,7 +757,7 @@ def duplicated(values, keep='first'):
     duplicated : ndarray
     """
 
-    values, dtype, ndtype = _ensure_data(values)
+    values, dtype, ndtype, inferred = _ensure_data(values)
     f = getattr(htable, "duplicated_{dtype}".format(dtype=ndtype))
     return f(values, keep=keep)
 
@@ -741,7 +787,7 @@ def mode(values):
             return Series(values.values.mode(), name=values.name)
         return values.mode()
 
-    values, dtype, ndtype = _ensure_data(values)
+    values, dtype, ndtype, inferred = _ensure_data(values)
 
     # TODO: this should support float64
     if ndtype not in ['int64', 'uint64', 'object']:
@@ -785,11 +831,11 @@ def rank(values, axis=0, method='average', na_option='keep',
         (e.g. 1, 2, 3) or in percentile form (e.g. 0.333..., 0.666..., 1).
     """
     if values.ndim == 1:
-        f, values = _get_data_algo(values, _rank1d_functions)
+        f, values, _ = _get_data_algo(values, _rank1d_functions)
         ranks = f(values, ties_method=method, ascending=ascending,
                   na_option=na_option, pct=pct)
     elif values.ndim == 2:
-        f, values = _get_data_algo(values, _rank2d_functions)
+        f, values, _ = _get_data_algo(values, _rank2d_functions)
         ranks = f(values, axis=axis, ties_method=method,
                   ascending=ascending, na_option=na_option, pct=pct)
     else:
@@ -1049,7 +1095,7 @@ def compute(self, method):
             return dropped[slc].sort_values(ascending=ascending).head(n)
 
         # fast method
-        arr, _, _ = _ensure_data(dropped.values)
+        arr, _, _, _ = _ensure_data(dropped.values)
         if method == 'nlargest':
             arr = -arr
 

pandas/core/categorical.py

@@ -2088,8 +2088,9 @@ def _get_codes_for_values(values, categories):
         values = _ensure_object(values)
         categories = _ensure_object(categories)
 
-    (hash_klass, vec_klass), vals = _get_data_algo(values, _hashtables)
-    (_, _), cats = _get_data_algo(categories, _hashtables)
+    (hash_klass, vec_klass), vals, dtype = _get_data_algo(
+        values, _hashtables, infer=False)
+    (_, _), cats, _ = _get_data_algo(categories, _hashtables, dtype=dtype)
     t = hash_klass(len(cats))
     t.map_locations(cats)
     return coerce_indexer_dtype(t.lookup(vals), cats)

pandas/tests/indexes/common.py

@@ -323,6 +323,7 @@ def test_get_unique_index(self):
 
             idx = ind[[0] * 5]
             idx_unique = ind[[0]]
+
             # We test against `idx_unique`, so first we make sure it's unique
             # and doesn't contain nans.
             self.assertTrue(idx_unique.is_unique)
@@ -336,7 +337,6 @@ def test_get_unique_index(self):
                 tm.assert_index_equal(result, idx_unique)
 
             # nans:
-
             if not ind._can_hold_na:
                 continue
 

pandas/tests/test_algos.py

@@ -387,6 +387,12 @@ def test_uint64_overflow(self):
         exp = np.array([1, 2, 2**63], dtype=np.uint64)
         tm.assert_numpy_array_equal(algos.unique(s), exp)
 
+    def test_nan_in_object_array(self):
+        l = ['a', np.nan, 'c', 'c']
+        result = pd.unique(l)
+        expected = np.array(['a', np.nan, 'c'], dtype=object)
+        tm.assert_numpy_array_equal(result, expected)
+
     def test_categorical(self):
 
         # we are expecting to return in the order
@@ -1375,11 +1381,11 @@ def test_int64_add_overflow():
 class TestMode(tm.TestCase):
 
     def test_no_mode(self):
-        exp = Series([], dtype=np.float64)
+        exp = Series([], dtype=object)
         tm.assert_series_equal(algos.mode([]), exp)
 
-    # GH 15714
     def test_mode_single(self):
+        # GH 15714
         exp_single = [1]
         data_single = [1]