From 930a6ec13e61926348fa6848314805db8c76f3e6 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Wed, 1 Apr 2020 18:10:38 -0700 Subject: [PATCH 1/3] REF: sql insert_data operate column-wise to avoid internals --- pandas/io/sql.py | 41 +++++++++++++++-------------------------- 1 file changed, 15 insertions(+), 26 deletions(-) diff --git a/pandas/io/sql.py b/pandas/io/sql.py index 3891afa5e332d..48fabe76e9e1c 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -692,37 +692,26 @@ def insert_data(self): column_names = list(map(str, temp.columns)) ncols = len(column_names) data_list = [None] * ncols - blocks = temp._data.blocks - - for b in blocks: - if b.is_datetime: - # return datetime.datetime objects - if b.is_datetimetz: - # GH 9086: Ensure we return datetimes with timezone info - # Need to return 2-D data; DatetimeIndex is 1D - d = b.values.to_pydatetime() - d = np.atleast_2d(d) - else: - # convert to microsecond resolution for datetime.datetime - d = b.values.astype("M8[us]").astype(object) - elif b.is_timedelta: - # numpy converts this to an object array of integers, - # whereas b.astype(object).values would convert to - # object array of Timedeltas - d = b.values.astype(object) + + for i in range(len(temp.columns)): + ser = temp.iloc[:, i] + vals = ser._values + if vals.dtype.kind == "M": + d = vals.to_pydatetime() + elif vals.dtype.kind == "m": + # store as integers, see GH#6921, GH#7076 + d = vals.view("i8").astype(object) else: - # TODO(2DEA): astype-first can be avoided with 2D EAs - # astype on the block instead of values to ensure we - # get the right shape - d = b.astype(object).values + d = vals.astype(object) + + assert isinstance(d, np.ndarray), type(d) - # replace NaN with None - if b._can_hold_na: + if ser._can_hold_na: + # Note: this will miss timedeltas since they are converted to int mask = isna(d) d[mask] = None - for col_loc, col in zip(b.mgr_locs, d): - data_list[col_loc] = col + data_list[i] = d return column_names, data_list From 8834ac1be10f12d8198c868e4f326031a80effd5 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Thu, 2 Apr 2020 14:40:11 -0700 Subject: [PATCH 2/3] iloc->_ixs --- pandas/io/sql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/io/sql.py b/pandas/io/sql.py index 48fabe76e9e1c..5fe3dab6abd5f 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -694,7 +694,7 @@ def insert_data(self): data_list = [None] * ncols for i in range(len(temp.columns)): - ser = temp.iloc[:, i] + ser = temp._ixs(i, axis=1) # i.e. temp.iloc[:, i] vals = ser._values if vals.dtype.kind == "M": d = vals.to_pydatetime() From eae5b1ee0e1dcb57f1414d8612f82c249eea28b0 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Mon, 6 Apr 2020 16:23:27 -0700 Subject: [PATCH 3/3] use items --- pandas/io/sql.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pandas/io/sql.py b/pandas/io/sql.py index 48fabe76e9e1c..c657a925a5eab 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -693,8 +693,7 @@ def insert_data(self): ncols = len(column_names) data_list = [None] * ncols - for i in range(len(temp.columns)): - ser = temp.iloc[:, i] + for i, (_, ser) in enumerate(temp.items()): vals = ser._values if vals.dtype.kind == "M": d = vals.to_pydatetime()