ENH: memoize objects when reading from file to reduce memory footprint

Author: wesm

pandas/io/parsers.py

@@ -219,7 +219,7 @@ def __init__(self, f, delimiter=None, names=None, header=0,
         self.chunksize = chunksize
         self.passed_names = names is not None
         self.encoding = encoding
-        
+
 
         if com.is_integer(skiprows):
             skiprows = range(skiprows)
@@ -281,7 +281,7 @@ def _make_reader(self, f):
                                                     dialect=dia)))
 
             if self.encoding is not None:
-                reader = com.UnicodeReader(f, dialect=dia, 
+                reader = com.UnicodeReader(f, dialect=dia,
                                            encoding=self.encoding)
             else:
                 reader = csv.reader(f, dialect=dia)

pandas/src/inference.pyx

@@ -341,6 +341,7 @@ def sanitize_objects(ndarray[object] values, set na_values):
         Py_ssize_t i, n
         object val, onan
         Py_ssize_t na_count = 0
+        dict memo = {}
 
     n = len(values)
     onan = np.nan
@@ -350,6 +351,11 @@ def sanitize_objects(ndarray[object] values, set na_values):
         if val == '' or val in na_values:
             values[i] = onan
             na_count += 1
+        elif val in memo:
+            values[i] = memo[val]
+        else:
+            memo[val] = val
+
     return na_count
 
 def maybe_convert_bool(ndarray[object] arr):