BUG/TST: replace iterrows with itertuples in sql insert (GH6509)

Author: jorisvandenbossche

doc/source/release.rst

@@ -227,6 +227,8 @@ Bug Fixes
 - Series.quantile raising on an ``object`` dtype (:issue:`6555`)
 - Bug in ``.xs`` with a ``nan`` in level when dropped (:issue:`6574`)
 - Bug in fillna with method = 'bfill/ffill' and ``datetime64[ns]`` dtype (:issue:`6587`)
+- Bug in sql writing with mixed dtypes possibly leading to data loss (:issue:`6509`)
+
 
 pandas 0.13.1
 -------------

pandas/io/sql.py

@@ -423,16 +423,17 @@ def insert(self):
         ins = self.insert_statement()
         data_list = []
         # to avoid if check for every row
+        keys = self.frame.columns
         if self.index is not None:
-            for t in self.frame.iterrows():
+            for t in self.frame.itertuples():
                 data = dict((k, self.maybe_asscalar(v))
-                            for k, v in t[1].iteritems())
+                            for k, v in zip(keys, t[1:]))
                 data[self.index] = self.maybe_asscalar(t[0])
                 data_list.append(data)
         else:
-            for t in self.frame.iterrows():
+            for t in self.frame.itertuples():
                 data = dict((k, self.maybe_asscalar(v))
-                            for k, v in t[1].iteritems())
+                            for k, v in zip(keys, t[1:]))
                 data_list.append(data)
         self.pd_sql.execute(ins, data_list)
 
@@ -758,8 +759,8 @@ def insert_statement(self):
     def insert(self):
         ins = self.insert_statement()
         cur = self.pd_sql.con.cursor()
-        for r in self.frame.iterrows():
-            data = [self.maybe_asscalar(v) for v in r[1].values]
+        for r in self.frame.itertuples():
+            data = [self.maybe_asscalar(v) for v in r[1:]]
             if self.index is not None:
                 data.insert(0, self.maybe_asscalar(r[0]))
             cur.execute(ins, tuple(data))

pandas/io/tests/test_sql.py

@@ -7,7 +7,7 @@
 import nose
 import numpy as np
 
-from pandas import DataFrame
+from pandas import DataFrame, Series
 from pandas.compat import range, lrange, iteritems
 #from pandas.core.datetools import format as date_format
 
@@ -554,6 +554,18 @@ def test_date_parsing(self):
         self.assertTrue(issubclass(df.IntDateCol.dtype.type, np.datetime64),
                         "IntDateCol loaded with incorrect type")
 
+    def test_mixed_dtype_insert(self):
+        # see GH6509
+        s1 = Series(2**25 + 1,dtype=np.int32)
+        s2 = Series(0.0,dtype=np.float32)
+        df = DataFrame({'s1': s1, 's2': s2})
+
+        # write and read again        
+        df.to_sql("test_read_write", self.conn)
+        df2 = sql.read_table("test_read_write", self.conn)
+
+        tm.assert_equal(df['s1'].values, df2['s1'].values)         
+
 
 class TestSQLAlchemy(_TestSQLAlchemy):
     """