ENH: add explicit duplicate check when creating an index in parsing functions,

further address concerns raised in GH #226

Author: wesm

RELEASE.rst

@@ -15,6 +15,8 @@ pandas 0.4.4
   - Added `parse_dates` option to `read_csv` and `read_table` methods to
     optionally try to parse dates in the index columns
   - Added ability to join on multiple columns in `DataFrame.join` (GH #214)
+  - Added private `_get_duplicates` function to `Index` for identifying
+    duplicate values more easily
 
 **API Changes**
 
@@ -25,6 +27,9 @@ pandas 0.4.4
 
 **Improvements to existing features**
 
+  - File parsing functions like `read_csv` and `read_table` will explicitly
+    check if a parsed index has duplicates and raise a more helpful exception
+    rather than deferring the check until later
   - Refactored merging / joining code into a tidy class and disabled unnecessary
     computations in the float/object case, thus getting about 10% better
     performance

pandas/core/frame.py

@@ -2125,7 +2125,7 @@ def append(self, other, ignore_index=False):
             new_index = None
         else:
             new_index = self.index.append(other.index)
-            new_index._verify_integrity()
+            assert(new_index._verify_integrity())
 
         if self.columns.equals(other.columns):
             return self._append_same_columns(other, new_index)

pandas/core/index.py

@@ -99,8 +99,20 @@ def indexMap(self):
         return self._indexMap
 
     def _verify_integrity(self):
+        if self._indexMap is None:
+            try:
+                self.indexMap
+            except Exception:
+                return False
         return len(self.indexMap) == len(self)
 
+    def _get_duplicates(self):
+        from collections import defaultdict
+        counter = defaultdict(lambda: 0)
+        for k in self.values:
+            counter[k] += 1
+        return sorted(k for k, v in counter.iteritems() if v > 1)
+
     _allDates = None
     def is_all_dates(self):
         """

pandas/io/parsers.py

@@ -196,12 +196,17 @@ def _simple_parser(lines, colNames=None, header=0, index_col=0,
                 index = _try_parse_dates(index, parser=date_parser)
             index = Index(_maybe_convert_int(np.array(index, dtype=object)))
         else:
-            index = MultiIndex.from_arrays(_maybe_convert_int_mindex(index,
-                                                 parse_dates, date_parser),
+            arrays = _maybe_convert_int_mindex(index, parse_dates,
+                                               date_parser)
+            index = MultiIndex.from_arrays(arrays,
                                                  names=idx_names)
     else:
         index = Index(np.arange(len(content)))
 
+    if not index._verify_integrity():
+        dups = index._get_duplicates()
+        raise Exception('Index has duplicates: %s' % str(dups))
+
     if len(columns) != len(zipped_content):
         raise Exception('wrong number of columns')
 

pandas/io/tests/test_parsers.py

@@ -140,7 +140,16 @@ def test_read_table_wrong_num_columns(self):
 """
         self.assertRaises(Exception, read_csv, StringIO(data))
 
-
+    def test_read_table_duplicate_index(self):
+        data = """index,A,B,C,D
+foo,2,3,4,5
+bar,7,8,9,10
+baz,12,13,14,15
+qux,12,13,14,15
+foo,12,13,14,15
+bar,12,13,14,15
+"""
+        self.assertRaises(Exception, read_csv, StringIO(data), index_col=0)
 
 def curpath():
     pth, _ = os.path.split(os.path.abspath(__file__))

pandas/tests/test_index.py

@@ -33,7 +33,7 @@ def test_deepcopy(self):
 
     def test_duplicates(self):
         idx = Index([0, 0, 0])
-        self.assertRaises(Exception, idx._verify_integrity)
+        self.assert_(not idx._verify_integrity())
 
     def test_sort(self):
         self.assertRaises(Exception, self.strIndex.sort)