fix for duplicate cols in select_dtypes and get_dummies

Author: kunalgosar

pandas/core/frame.py

@@ -2991,15 +2991,15 @@ def select_dtypes(self, include=None, exclude=None):
         include_these = Series(not bool(include), index=self.columns)
         exclude_these = Series(not bool(exclude), index=self.columns)
 
-        def is_dtype_instance_mapper(column, dtype):
-            return column, functools.partial(issubclass, dtype.type)
+        def is_dtype_instance_mapper(idx, dtype):
+            return idx, functools.partial(issubclass, dtype.type)
 
-        for column, f in itertools.starmap(is_dtype_instance_mapper,
-                                           self.dtypes.iteritems()):
+        for idx, f in itertools.starmap(is_dtype_instance_mapper,
+                                        enumerate(self.dtypes)):
             if include:  # checks for the case of empty include or exclude
-                include_these[column] = any(map(f, include))
+                include_these.iloc[idx] = any(map(f, include))
             if exclude:
-                exclude_these[column] = not any(map(f, exclude))
+                exclude_these.iloc[idx] = not any(map(f, exclude))
 
         dtype_indexer = include_these & exclude_these
         return self.loc[com._get_info_slice(self, dtype_indexer)]

pandas/core/reshape/reshape.py

@@ -826,45 +826,49 @@ def get_dummies(data, prefix=None, prefix_sep='_', dummy_na=False,
 
         if columns is None:
             columns_to_encode = data.select_dtypes(
-                include=['object', 'category']).columns
+                include=['object', 'category'])
         else:
-            columns_to_encode = columns
+            columns_to_encode = data[columns]
 
         # validate prefixes and separator to avoid silently dropping cols
         def check_len(item, name):
             len_msg = ("Length of '{name}' ({len_item}) did not match the "
                        "length of the columns being encoded ({len_enc}).")
 
             if is_list_like(item):
-                if not len(item) == len(columns_to_encode):
-                    len_msg = len_msg.format(name=name, len_item=len(item),
-                                             len_enc=len(columns_to_encode))
+                if not len(item) == columns_to_encode.shape[1]:
+                    len_msg = \
+                        len_msg.format(name=name, len_item=len(item),
+                                       len_enc=columns_to_encode.shape[1])
                     raise ValueError(len_msg)
 
         check_len(prefix, 'prefix')
         check_len(prefix_sep, 'prefix_sep')
         if isinstance(prefix, compat.string_types):
             prefix = cycle([prefix])
         if isinstance(prefix, dict):
-            prefix = [prefix[col] for col in columns_to_encode]
+            prefix = [prefix[col] for col in columns_to_encode.columns]
 
         if prefix is None:
-            prefix = columns_to_encode
+            prefix = columns_to_encode.columns
 
         # validate separators
         if isinstance(prefix_sep, compat.string_types):
             prefix_sep = cycle([prefix_sep])
         elif isinstance(prefix_sep, dict):
-            prefix_sep = [prefix_sep[col] for col in columns_to_encode]
+            prefix_sep = [prefix_sep[col] for col in columns_to_encode.columns]
 
-        if set(columns_to_encode) == set(data.columns):
+        if columns_to_encode.shape == data.shape:
             with_dummies = []
+        elif columns is not None:
+            with_dummies = [data.drop(columns, axis=1)]
         else:
-            with_dummies = [data.drop(columns_to_encode, axis=1)]
+            with_dummies = [data.select_dtypes(exclude=['object', 'category'])]
 
-        for (col, pre, sep) in zip(columns_to_encode, prefix, prefix_sep):
+        for (col, pre, sep) in zip(columns_to_encode.iteritems(), prefix,
+                                   prefix_sep):
 
-            dummy = _get_dummies_1d(data[col], prefix=pre, prefix_sep=sep,
+            dummy = _get_dummies_1d(col[1], prefix=pre, prefix_sep=sep,
                                     dummy_na=dummy_na, sparse=sparse,
                                     drop_first=drop_first, dtype=dtype)
             with_dummies.append(dummy)