BUG: Bug with reindexing where a non-unique index will now raise ValueError (GH4746)

Author: jreback

doc/source/release.rst

@@ -326,6 +326,7 @@ See :ref:`Internal Refactoring<whatsnew_0130.refactoring>`
   - Bug with using ``QUOTE_NONE`` with ``to_csv`` causing ``Exception``. (:issue:`4328`)
   - Bug with Series indexing not raising an error when the right-hand-side has an incorrect length (:issue:`2702`)
   - Bug in multi-indexing with a partial string selection as one part of a MultIndex (:issue:`4758`)
+  - Bug with reindexing on the index with a non-unique index will now raise ``ValueError`` (:issue:`4746`)
 
 pandas 0.12
 ===========

pandas/core/frame.py

@@ -2267,15 +2267,15 @@ def _reindex_index(self, new_index, method, copy, level, fill_value=NA,
                                                 limit=limit, copy_if_needed=True,
                                                 takeable=takeable)
         return self._reindex_with_indexers({0: [new_index, indexer]},
-                                           copy=copy, fill_value=fill_value)
+                                           copy=copy, fill_value=fill_value, allow_dups=takeable)
 
     def _reindex_columns(self, new_columns, copy, level, fill_value=NA,
                          limit=None, takeable=False):
         new_columns, indexer = self.columns.reindex(new_columns, level=level,
                                                     limit=limit, copy_if_needed=True,
                                                     takeable=takeable)
         return self._reindex_with_indexers({1: [new_columns, indexer]},
-                                           copy=copy, fill_value=fill_value)
+                                           copy=copy, fill_value=fill_value, allow_dups=takeable)
 
     def _reindex_multi(self, axes, copy, fill_value):
         """ we are guaranteed non-Nones in the axes! """
@@ -2541,8 +2541,7 @@ def take(self, indices, axis=0, convert=True):
                 new_data = self._data.take(indices, axis=1, verify=False)
                 return DataFrame(new_data)
             else:
-                new_columns = self.columns.take(indices)
-                return self.reindex(columns=new_columns)
+                return self.reindex(columns=indices, takeable=True)
         else:
             new_values = com.take_nd(self.values,
                                      com._ensure_int64(indices),

pandas/core/generic.py

@@ -944,7 +944,7 @@ def drop(self, labels, axis=0, level=None):
                 new_axis = axis.drop(labels, level=level)
             else:
                 new_axis = axis.drop(labels)
-            dropped = self.reindex(**{axis_name: new_axis})
+            dropped = self.reindex(**{ axis_name: new_axis })
             try:
                 dropped.axes[axis_].set_names(axis.names, inplace=True)
             except AttributeError:
@@ -1161,7 +1161,8 @@ def reindex_axis(self, labels, axis=0, method=None, level=None, copy=True,
         return self._reindex_with_indexers({axis: [new_index, indexer]}, method=method, fill_value=fill_value,
                                            limit=limit, copy=copy)._propogate_attributes(self)
 
-    def _reindex_with_indexers(self, reindexers, method=None, fill_value=np.nan, limit=None, copy=False):
+    def _reindex_with_indexers(self, reindexers, method=None, fill_value=np.nan, limit=None, copy=False, allow_dups=False):
+        """ allow_dups indicates an internal call here """
 
         # reindex doing multiple operations on different axes if indiciated
         new_data = self._data
@@ -1183,7 +1184,7 @@ def _reindex_with_indexers(self, reindexers, method=None, fill_value=np.nan, lim
                 # TODO: speed up on homogeneous DataFrame objects
                 indexer = com._ensure_int64(indexer)
                 new_data = new_data.reindex_indexer(index, indexer, axis=baxis,
-                                                    fill_value=fill_value)
+                                                    fill_value=fill_value, allow_dups=allow_dups)
 
             elif baxis == 0 and index is not None and index is not new_data.axes[baxis]:
                 new_data = new_data.reindex_items(index, copy=copy,

pandas/core/indexing.py

@@ -668,7 +668,7 @@ def _reindex(keys, level=None):
                     if axis+1 > ndim:
                         raise AssertionError("invalid indexing error with non-unique index")
 
-                    result = result._reindex_with_indexers({ axis : [ new_labels, new_indexer ] }, copy=True)
+                    result = result._reindex_with_indexers({ axis : [ new_labels, new_indexer ] }, copy=True, allow_dups=True)
 
                 return result
 

pandas/core/internals.py

@@ -198,6 +198,7 @@ def reindex_axis(self, indexer, method=None, axis=1, fill_value=None, limit=None
             raise AssertionError('axis must be at least 1, got %d' % axis)
         if fill_value is None:
             fill_value = self.fill_value
+
         new_values = com.take_nd(self.values, indexer, axis,
                                  fill_value=fill_value, mask_info=mask_info)
         return make_block(
@@ -2718,10 +2719,14 @@ def reindex_axis0_with_method(self, new_axis, indexer=None, method=None, fill_va
         raise AssertionError('method argument not supported for '
                              'axis == 0')
 
-    def reindex_indexer(self, new_axis, indexer, axis=1, fill_value=None):
+    def reindex_indexer(self, new_axis, indexer, axis=1, fill_value=None, allow_dups=False):
         """
         pandas-indexer with -1's only.
         """
+        # trying to reindex on an axis with duplicates
+        if not allow_dups and not self.axes[axis].is_unique:
+            raise ValueError("cannot reindex from a duplicate axis")
+
         if axis == 0:
             return self._reindex_indexer_items(new_axis, indexer, fill_value)
 

pandas/tests/test_frame.py

@@ -2879,7 +2879,7 @@ def test_constructor_column_duplicates(self):
                           columns=['b', 'a', 'a'])
 
 
-    def test_column_duplicates_operations(self):
+    def test_column_dups_operations(self):
 
         def check(result, expected=None):
             if expected is not None:
@@ -2973,22 +2973,6 @@ def check(result, expected=None):
         expected = DataFrame([[1,5,7.],[1,5,7.],[1,5,7.]],columns=['bar','hello','foo2'])
         check(df,expected)
 
-        # reindex
-        df = DataFrame([[1,5,7.],[1,5,7.],[1,5,7.]],columns=['bar','a','a'])
-        expected = DataFrame([[1],[1],[1]],columns=['bar'])
-        result = df.reindex(columns=['bar'])
-        check(result,expected)
-
-        result1 = DataFrame([[1],[1],[1]],columns=['bar']).reindex(columns=['bar','foo'])
-        result2 = df.reindex(columns=['bar','foo'])
-        check(result2,result1)
-
-        # drop
-        df = DataFrame([[1,5,7.],[1,5,7.],[1,5,7.]],columns=['bar','a','a'])
-        df = df.drop(['a'],axis=1)
-        expected = DataFrame([[1],[1],[1]],columns=['bar'])
-        check(df,expected)
-
         # values
         df = DataFrame([[1,2.5],[3,4.5]], index=[1,2], columns=['x','x'])
         result = df.values
@@ -3016,6 +3000,17 @@ def check(result, expected=None):
                              columns=['RT','TClose','TExg','RPT_Date','STK_ID','STK_Name','QT_Close']).set_index(['STK_ID','RPT_Date'],drop=False)
         assert_frame_equal(result,expected)
 
+        # reindex is invalid!
+        df = DataFrame([[1,5,7.],[1,5,7.],[1,5,7.]],columns=['bar','a','a'])
+        self.assertRaises(ValueError, df.reindex, columns=['bar'])
+        self.assertRaises(ValueError, df.reindex, columns=['bar','foo'])
+
+        # drop
+        df = DataFrame([[1,5,7.],[1,5,7.],[1,5,7.]],columns=['bar','a','a'])
+        df = df.drop(['a'],axis=1)
+        expected = DataFrame([[1],[1],[1]],columns=['bar'])
+        check(df,expected)
+
     def test_insert_benchmark(self):
         # from the vb_suite/frame_methods/frame_insert_columns
         N = 10
@@ -7573,6 +7568,21 @@ def test_reindex_fill_value(self):
         expected = df.reindex(lrange(15)).fillna(0)
         assert_frame_equal(result, expected)
 
+    def test_reindex_dups(self):
+
+        # GH4746, reindex on duplicate index error messages
+        arr = np.random.randn(10)
+        df = DataFrame(arr,index=[1,2,3,4,5,1,2,3,4,5])
+
+        # set index is ok
+        result = df.copy()
+        result.index = list(range(len(df)))
+        expected = DataFrame(arr,index=list(range(len(df))))
+        assert_frame_equal(result,expected)
+
+        # reindex fails
+        self.assertRaises(ValueError, df.reindex, index=list(range(len(df))))
+
     def test_align(self):
 
         af, bf = self.frame.align(self.frame)