From a77cdfd63df72c8b4d70783b12c0d9e4b748b9a7 Mon Sep 17 00:00:00 2001 From: Eduardo Schettino Date: Fri, 8 Jun 2018 14:52:01 +0800 Subject: [PATCH 01/16] ENH: to_sql() add parameter "method" to control insertions method (#8953) --- doc/source/whatsnew/v0.24.0.txt | 1 + pandas/core/generic.py | 54 +++++++++++++++++++++++++++-- pandas/io/sql.py | 61 +++++++++++++++++++++++++-------- pandas/tests/io/test_sql.py | 35 +++++++++++++++++-- 4 files changed, 131 insertions(+), 20 deletions(-) diff --git a/doc/source/whatsnew/v0.24.0.txt b/doc/source/whatsnew/v0.24.0.txt index 88a333b619141..28262fb7c5511 100644 --- a/doc/source/whatsnew/v0.24.0.txt +++ b/doc/source/whatsnew/v0.24.0.txt @@ -16,6 +16,7 @@ Other Enhancements - :func:`Series.mode` and :func:`DataFrame.mode` now support the ``dropna`` parameter which can be used to specify whether NaN/NaT values should be considered (:issue:`17534`) - :func:`to_csv` now supports ``compression`` keyword when a file handle is passed. (:issue:`21227`) - :meth:`Index.droplevel` is now implemented also for flat indexes, for compatibility with MultiIndex (:issue:`21115`) +- :func:`~pandas.DataFrame.to_sql` add parameter ``method`` to control SQL insertion clause (:8953:) .. _whatsnew_0240.api_breaking: diff --git a/pandas/core/generic.py b/pandas/core/generic.py index 32f64b1d3e05c..83e52eacdfe58 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -2014,7 +2014,7 @@ def to_msgpack(self, path_or_buf=None, encoding='utf-8', **kwargs): **kwargs) def to_sql(self, name, con, schema=None, if_exists='fail', index=True, - index_label=None, chunksize=None, dtype=None): + index_label=None, chunksize=None, dtype=None, method='default'): """ Write records stored in a DataFrame to a SQL database. @@ -2052,6 +2052,8 @@ def to_sql(self, name, con, schema=None, if_exists='fail', index=True, Specifying the datatype for columns. The keys should be the column names and the values should be the SQLAlchemy types or strings for the sqlite3 legacy mode. + method : {'default', 'multi', callable}, default 'default' + Controls the SQL insertion clause used. Raises ------ @@ -2120,11 +2122,59 @@ def to_sql(self, name, con, schema=None, if_exists='fail', index=True, >>> engine.execute("SELECT * FROM integers").fetchall() [(1,), (None,), (2,)] + + Insertion method: + + .. versionadded:: 0.24.0 + + The parameter ``method`` controls the SQL insertion clause used. + Possible values are: + + - `'default'`: Uses standard SQL `INSERT` clause + - `'multi'`: Pass multiple values in a single `INSERT` clause. + It uses a **special** SQL syntax not supported by all backends. + This usually provides a big performance for Analytic databases + like *Presto* and *Redshit*, but has worse performance for + traditional SQL backend if the table contains many columns. + For more information check SQLAlchemy `documention `__. + - callable: with signature `(pd_table, conn, keys, data_iter)`. + This can be used to implement more performant insertion based on + specific backend dialect features. + I.e. using *Postgresql* `COPY clause + `__. + Check API for details and a sample implementation + :func:`~pandas.DataFrame.to_sql`. + + + Example of callable for Postgresql *COPY*:: + + # Alternative to_sql() *method* for DBs that support COPY FROM + import csv + from io import StringIO + + def psql_insert_copy(table, conn, keys, data_iter): + # gets a DBAPI connection that can provide a cursor + dbapi_conn = conn.connection + with dbapi_conn.cursor() as cur: + s_buf = StringIO() + writer = csv.writer(s_buf) + writer.writerows(data_iter) + s_buf.seek(0) + + columns = ', '.join('"{}"'.format(k) for k in keys) + if table.schema: + table_name = '{}.{}'.format(table.schema, table.name) + else: + table_name = table.name + + sql = 'COPY {} ({}) FROM STDIN WITH CSV'.format( + table_name, columns) + cur.copy_expert(sql=sql, file=s_buf) """ from pandas.io import sql sql.to_sql(self, name, con, schema=schema, if_exists=if_exists, index=index, index_label=index_label, chunksize=chunksize, - dtype=dtype) + dtype=dtype, method=method) def to_pickle(self, path, compression='infer', protocol=pkl.HIGHEST_PROTOCOL): diff --git a/pandas/io/sql.py b/pandas/io/sql.py index a582d32741ae9..a0a21ae0ae4f0 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -6,6 +6,7 @@ from __future__ import print_function, division from datetime import datetime, date, time +from functools import partial import warnings import re @@ -398,7 +399,7 @@ def read_sql(sql, con, index_col=None, coerce_float=True, params=None, def to_sql(frame, name, con, schema=None, if_exists='fail', index=True, - index_label=None, chunksize=None, dtype=None): + index_label=None, chunksize=None, dtype=None, method='default'): """ Write records stored in a DataFrame to a SQL database. @@ -432,6 +433,8 @@ def to_sql(frame, name, con, schema=None, if_exists='fail', index=True, Optional specifying the datatype for columns. The SQL type should be a SQLAlchemy type, or a string for sqlite3 fallback connection. If all columns are of the same type, one single value can be used. + method : {'default', 'multi', callable}, default 'default' + Controls the SQL insertion clause used. """ if if_exists not in ('fail', 'replace', 'append'): @@ -447,7 +450,7 @@ def to_sql(frame, name, con, schema=None, if_exists='fail', index=True, pandas_sql.to_sql(frame, name, if_exists=if_exists, index=index, index_label=index_label, schema=schema, - chunksize=chunksize, dtype=dtype) + chunksize=chunksize, dtype=dtype, method=method) def has_table(table_name, con, schema=None): @@ -572,8 +575,25 @@ def create(self): else: self._execute_create() - def insert_statement(self): - return self.table.insert() + def _execute_insert(self, conn, keys, data_iter): + """Execute SQL statement inserting data + + Parameters + ---------- + data : list of list + of values to be inserted + """ + data = [{k: v for k, v in zip(keys, row)} for row in data_iter] + conn.execute(self.table.insert(), data) + + def _execute_insert_multi(self, conn, keys, data_iter): + """Alternative to _exec_insert for DBs that support multivalue INSERT. + + Note: multi-value insert is usually faster for a few columns + but performance degrades quickly with increase of columns. + """ + data = [{k: v for k, v in zip(keys, row)} for row in data_iter] + conn.execute(self.table.insert(data)) def insert_data(self): if self.index is not None: @@ -611,11 +631,18 @@ def insert_data(self): return column_names, data_list - def _execute_insert(self, conn, keys, data_iter): - data = [{k: v for k, v in zip(keys, row)} for row in data_iter] - conn.execute(self.insert_statement(), data) + def insert(self, chunksize=None, method=None): + + # set insert method + if method in (None, 'default'): + exec_insert = self._execute_insert + elif method == 'multi': + exec_insert = self._execute_insert_multi + elif callable(method): + exec_insert = partial(method, self) + else: + raise ValueError('Invalid parameter `method`: {}'.format(method)) - def insert(self, chunksize=None): keys, data_list = self.insert_data() nrows = len(self.frame) @@ -638,7 +665,7 @@ def insert(self, chunksize=None): break chunk_iter = zip(*[arr[start_i:end_i] for arr in data_list]) - self._execute_insert(conn, keys, chunk_iter) + exec_insert(conn, keys, chunk_iter) def _query_iterator(self, result, chunksize, columns, coerce_float=True, parse_dates=None): @@ -1078,7 +1105,8 @@ def read_query(self, sql, index_col=None, coerce_float=True, read_sql = read_query def to_sql(self, frame, name, if_exists='fail', index=True, - index_label=None, schema=None, chunksize=None, dtype=None): + index_label=None, schema=None, chunksize=None, dtype=None, + method='default'): """ Write records stored in a DataFrame to a SQL database. @@ -1108,7 +1136,8 @@ def to_sql(self, frame, name, if_exists='fail', index=True, Optional specifying the datatype for columns. The SQL type should be a SQLAlchemy type. If all columns are of the same type, one single value can be used. - + method : {'default', 'multi', callable}, default 'default' + Controls the SQL insertion clause used. """ if dtype and not is_dict_like(dtype): dtype = {col_name: dtype for col_name in frame} @@ -1124,7 +1153,7 @@ def to_sql(self, frame, name, if_exists='fail', index=True, if_exists=if_exists, index_label=index_label, schema=schema, dtype=dtype) table.create() - table.insert(chunksize) + table.insert(chunksize, method=method) if (not name.isdigit() and not name.islower()): # check for potentially case sensitivity issues (GH7815) # Only check when name is not a number and name is not lower case @@ -1434,7 +1463,8 @@ def _fetchall_as_list(self, cur): return result def to_sql(self, frame, name, if_exists='fail', index=True, - index_label=None, schema=None, chunksize=None, dtype=None): + index_label=None, schema=None, chunksize=None, dtype=None, + method='default'): """ Write records stored in a DataFrame to a SQL database. @@ -1463,7 +1493,8 @@ def to_sql(self, frame, name, if_exists='fail', index=True, Optional specifying the datatype for columns. The SQL type should be a string. If all columns are of the same type, one single value can be used. - + method : {'default', 'multi', callable}, default 'default' + Controls the SQL insertion clause used. """ if dtype and not is_dict_like(dtype): dtype = {col_name: dtype for col_name in frame} @@ -1478,7 +1509,7 @@ def to_sql(self, frame, name, if_exists='fail', index=True, if_exists=if_exists, index_label=index_label, dtype=dtype) table.create() - table.insert(chunksize) + table.insert(chunksize, method) def has_table(self, name, schema=None): # TODO(wesm): unused? diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index f3ab74d37a2bc..22844f568eded 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -372,12 +372,16 @@ def _read_sql_iris_named_parameter(self): iris_frame = self.pandasSQL.read_query(query, params=params) self._check_iris_loaded_frame(iris_frame) - def _to_sql(self): + def _to_sql(self, method=None): self.drop_table('test_frame1') - self.pandasSQL.to_sql(self.test_frame1, 'test_frame1') + self.pandasSQL.to_sql(self.test_frame1, 'test_frame1', method=method) assert self.pandasSQL.has_table('test_frame1') + num_entries = len(self.test_frame1) + num_rows = self._count_rows('test_frame1') + assert num_rows == num_entries + # Nuke table self.drop_table('test_frame1') @@ -431,6 +435,25 @@ def _to_sql_append(self): assert num_rows == num_entries self.drop_table('test_frame1') + def _to_sql_method_callable(self): + check = [] # used to double check function below is really being used + + def sample(pd_table, conn, keys, data_iter): + check.append(1) + data = [{k: v for k, v in zip(keys, row)} for row in data_iter] + conn.execute(pd_table.table.insert(), data) + self.drop_table('test_frame1') + + self.pandasSQL.to_sql(self.test_frame1, 'test_frame1', method=sample) + assert self.pandasSQL.has_table('test_frame1') + + assert check == [1] + num_entries = len(self.test_frame1) + num_rows = self._count_rows('test_frame1') + assert num_rows == num_entries + # Nuke table + self.drop_table('test_frame1') + def _roundtrip(self): self.drop_table('test_frame_roundtrip') self.pandasSQL.to_sql(self.test_frame1, 'test_frame_roundtrip') @@ -1180,7 +1203,7 @@ def setup_connect(self): pytest.skip( "Can't connect to {0} server".format(self.flavor)) - def test_aread_sql(self): + def test_read_sql(self): self._read_sql_iris() def test_read_sql_parameter(self): @@ -1204,6 +1227,12 @@ def test_to_sql_replace(self): def test_to_sql_append(self): self._to_sql_append() + def test_to_sql_method_multi(self): + self._to_sql(method='multi') + + def test_to_sql_method_callable(self): + self._to_sql_method_callable() + def test_create_table(self): temp_conn = self.connect() temp_frame = DataFrame( From 21e8c04a341d36d13cb8e9b016db34e8f1287903 Mon Sep 17 00:00:00 2001 From: Eduardo Schettino Date: Sat, 9 Jun 2018 19:15:07 +0800 Subject: [PATCH 02/16] ENH: to_sql() add parameter "method". Fix docstrings (#8953) --- pandas/io/sql.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pandas/io/sql.py b/pandas/io/sql.py index a0a21ae0ae4f0..5b4b7a9b5e1ec 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -580,16 +580,20 @@ def _execute_insert(self, conn, keys, data_iter): Parameters ---------- - data : list of list - of values to be inserted + conn : sqlalchemy.engine.Engine or sqlalchemy.engine.Connection + keys : list of str + Column names + data_iter : generator of list + Each item contains a list of values to be inserted """ data = [{k: v for k, v in zip(keys, row)} for row in data_iter] conn.execute(self.table.insert(), data) def _execute_insert_multi(self, conn, keys, data_iter): - """Alternative to _exec_insert for DBs that support multivalue INSERT. + """Alternative to _execute_insert for DBs support multivalue INSERT. - Note: multi-value insert is usually faster for a few columns + Note: multi-value insert is usually faster for analytics DBs + and tables containing a few columns but performance degrades quickly with increase of columns. """ data = [{k: v for k, v in zip(keys, row)} for row in data_iter] From 1e5d1cc203f494ade2f20ed477305e8de233e787 Mon Sep 17 00:00:00 2001 From: Eduardo Schettino Date: Mon, 11 Jun 2018 01:11:06 +0800 Subject: [PATCH 03/16] ENH: to_sql() add parameter "method". Improve docs based on reviews (#8953) --- doc/source/io.rst | 51 +++++++++++++++++++++++++++++++++++++ pandas/core/generic.py | 57 +++++++----------------------------------- pandas/io/sql.py | 30 ++++++++++++++++++++-- 3 files changed, 88 insertions(+), 50 deletions(-) diff --git a/doc/source/io.rst b/doc/source/io.rst index 32129147ee281..d217ef9189c00 100644 --- a/doc/source/io.rst +++ b/doc/source/io.rst @@ -4752,6 +4752,57 @@ default ``Text`` type for string columns: Because of this, reading the database table back in does **not** generate a categorical. +.. _io.sql.method: + +Insertion Method +++++++++++++++++ + +.. versionadded:: 0.24.0 + +The parameter ``method`` controls the SQL insertion clause used. + +Possible values are: + +- `'default'`: Uses standard SQL `INSERT` clause (one per row). +- `'multi'`: Pass multiple values in a single `INSERT` clause. + It uses a **special** SQL syntax not supported by all backends. + This usually provides better performance for Analytic databases + like *Presto* and *Redshit*, but has worse performance for + traditional SQL backend if the table contains many columns. + For more information check SQLAlchemy `documention + `__. +- callable: with signature `(pd_table, conn, keys, data_iter)`. + This can be used to implement more performant insertion based on + specific backend dialect features. + I.e. using *Postgresql* `COPY clause + `__. + +Example of callable for Postgresql *COPY*:: + + # Alternative to_sql() *method* for DBs that support COPY FROM + import csv + from io import StringIO + + def psql_insert_copy(table, conn, keys, data_iter): + # gets a DBAPI connection that can provide a cursor + dbapi_conn = conn.connection + with dbapi_conn.cursor() as cur: + s_buf = StringIO() + writer = csv.writer(s_buf) + writer.writerows(data_iter) + s_buf.seek(0) + + columns = ', '.join('"{}"'.format(k) for k in keys) + if table.schema: + table_name = '{}.{}'.format(table.schema, table.name) + else: + table_name = table.name + + sql = 'COPY {} ({}) FROM STDIN WITH CSV'.format( + table_name, columns) + cur.copy_expert(sql=sql, file=s_buf) + + Reading Tables '''''''''''''' diff --git a/pandas/core/generic.py b/pandas/core/generic.py index 83e52eacdfe58..9b8ad7ccff9eb 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -2055,6 +2055,15 @@ def to_sql(self, name, con, schema=None, if_exists='fail', index=True, method : {'default', 'multi', callable}, default 'default' Controls the SQL insertion clause used. + * `'default'`: Uses standard SQL `INSERT` clause (one per row). + * `'multi'`: Pass multiple values in a single `INSERT` clause. + * callable: with signature `(pd_table, conn, keys, data_iter)`. + + Details and a sample callable implementation on + section :ref:`insert method `. + + .. versionadded:: 0.24.0 + Raises ------ ValueError @@ -2122,54 +2131,6 @@ def to_sql(self, name, con, schema=None, if_exists='fail', index=True, >>> engine.execute("SELECT * FROM integers").fetchall() [(1,), (None,), (2,)] - - Insertion method: - - .. versionadded:: 0.24.0 - - The parameter ``method`` controls the SQL insertion clause used. - Possible values are: - - - `'default'`: Uses standard SQL `INSERT` clause - - `'multi'`: Pass multiple values in a single `INSERT` clause. - It uses a **special** SQL syntax not supported by all backends. - This usually provides a big performance for Analytic databases - like *Presto* and *Redshit*, but has worse performance for - traditional SQL backend if the table contains many columns. - For more information check SQLAlchemy `documention `__. - - callable: with signature `(pd_table, conn, keys, data_iter)`. - This can be used to implement more performant insertion based on - specific backend dialect features. - I.e. using *Postgresql* `COPY clause - `__. - Check API for details and a sample implementation - :func:`~pandas.DataFrame.to_sql`. - - - Example of callable for Postgresql *COPY*:: - - # Alternative to_sql() *method* for DBs that support COPY FROM - import csv - from io import StringIO - - def psql_insert_copy(table, conn, keys, data_iter): - # gets a DBAPI connection that can provide a cursor - dbapi_conn = conn.connection - with dbapi_conn.cursor() as cur: - s_buf = StringIO() - writer = csv.writer(s_buf) - writer.writerows(data_iter) - s_buf.seek(0) - - columns = ', '.join('"{}"'.format(k) for k in keys) - if table.schema: - table_name = '{}.{}'.format(table.schema, table.name) - else: - table_name = table.name - - sql = 'COPY {} ({}) FROM STDIN WITH CSV'.format( - table_name, columns) - cur.copy_expert(sql=sql, file=s_buf) """ from pandas.io import sql sql.to_sql(self, name, con, schema=schema, if_exists=if_exists, diff --git a/pandas/io/sql.py b/pandas/io/sql.py index 5b4b7a9b5e1ec..1a397a379d228 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -436,6 +436,14 @@ def to_sql(frame, name, con, schema=None, if_exists='fail', index=True, method : {'default', 'multi', callable}, default 'default' Controls the SQL insertion clause used. + * `'default'`: Uses standard SQL `INSERT` clause (one per row). + * `'multi'`: Pass multiple values in a single `INSERT` clause. + * callable: with signature `(pd_table, conn, keys, data_iter)`. + + Details and a sample callable implementation on + section :ref:`insert method `. + + .. versionadded:: 0.24.0 """ if if_exists not in ('fail', 'replace', 'append'): raise ValueError("'{0}' is not valid for if_exists".format(if_exists)) @@ -635,10 +643,10 @@ def insert_data(self): return column_names, data_list - def insert(self, chunksize=None, method=None): + def insert(self, chunksize=None, method='default'): # set insert method - if method in (None, 'default'): + if method == 'default': exec_insert = self._execute_insert elif method == 'multi': exec_insert = self._execute_insert_multi @@ -1142,6 +1150,15 @@ def to_sql(self, frame, name, if_exists='fail', index=True, single value can be used. method : {'default', 'multi', callable}, default 'default' Controls the SQL insertion clause used. + + * `'default'`: Uses standard SQL `INSERT` clause (one per row). + * `'multi'`: Pass multiple values in a single `INSERT` clause. + * callable: with signature `(pd_table, conn, keys, data_iter)`. + + Details and a sample callable implementation on + section :ref:`insert method `. + + .. versionadded:: 0.24.0 """ if dtype and not is_dict_like(dtype): dtype = {col_name: dtype for col_name in frame} @@ -1499,6 +1516,15 @@ def to_sql(self, frame, name, if_exists='fail', index=True, can be used. method : {'default', 'multi', callable}, default 'default' Controls the SQL insertion clause used. + + * `'default'`: Uses standard SQL `INSERT` clause (one per row). + * `'multi'`: Pass multiple values in a single `INSERT` clause. + * callable: with signature `(pd_table, conn, keys, data_iter)`. + + Details and a sample callable implementation on + section :ref:`insert method `. + + .. versionadded:: 0.24.0 """ if dtype and not is_dict_like(dtype): dtype = {col_name: dtype for col_name in frame} From 085313c4515666a40b792a4133efd292acfb34b4 Mon Sep 17 00:00:00 2001 From: Eduardo Schettino Date: Mon, 11 Jun 2018 11:08:32 +0800 Subject: [PATCH 04/16] ENH: to_sql() add parameter "method". Fix unit-test (#8953) --- pandas/tests/io/test_sql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 22844f568eded..1311cf50bc7b7 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -372,7 +372,7 @@ def _read_sql_iris_named_parameter(self): iris_frame = self.pandasSQL.read_query(query, params=params) self._check_iris_loaded_frame(iris_frame) - def _to_sql(self, method=None): + def _to_sql(self, method='default'): self.drop_table('test_frame1') self.pandasSQL.to_sql(self.test_frame1, 'test_frame1', method=method) From f4ffbfc3fc561d4de09b042dc0190188d07c045c Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Tue, 23 Oct 2018 23:02:45 +0200 Subject: [PATCH 05/16] doc clean-up --- doc/source/io.rst | 24 +++++++++++------------- pandas/io/sql.py | 30 +++++++++++++++--------------- 2 files changed, 26 insertions(+), 28 deletions(-) diff --git a/doc/source/io.rst b/doc/source/io.rst index 8fcab6f429ede..5f323cf7791ad 100644 --- a/doc/source/io.rst +++ b/doc/source/io.rst @@ -4814,24 +4814,22 @@ Insertion Method .. versionadded:: 0.24.0 The parameter ``method`` controls the SQL insertion clause used. - Possible values are: -- `'default'`: Uses standard SQL `INSERT` clause (one per row). -- `'multi'`: Pass multiple values in a single `INSERT` clause. - It uses a **special** SQL syntax not supported by all backends. - This usually provides better performance for Analytic databases - like *Presto* and *Redshit*, but has worse performance for +- ``'default'``: Uses standard SQL ``INSERT`` clause (one per row). +- ``'multi'``: Pass multiple values in a single ``INSERT`` clause. + It uses a *special* SQL syntax not supported by all backends. + This usually provides better performance for analytic databases + like *Presto* and *Redshift*, but has worse performance for traditional SQL backend if the table contains many columns. - For more information check SQLAlchemy `documention - `__. -- callable: with signature `(pd_table, conn, keys, data_iter)`. - This can be used to implement more performant insertion based on + For more information check the SQLAlchemy `documention + `__. +- callable with signature ``(pd_table, conn, keys, data_iter)``: + This can be used to implement a more performant insertion method based on specific backend dialect features. - I.e. using *Postgresql* `COPY clause - `__. -Example of callable for Postgresql *COPY*:: +Example of a callable using PostgreSQL `COPY clause +`__:: # Alternative to_sql() *method* for DBs that support COPY FROM import csv diff --git a/pandas/io/sql.py b/pandas/io/sql.py index 11459ebfcfa8e..8cb48ebe419d4 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -434,13 +434,13 @@ def to_sql(frame, name, con, schema=None, if_exists='fail', index=True, be a SQLAlchemy type, or a string for sqlite3 fallback connection. If all columns are of the same type, one single value can be used. method : {'default', 'multi', callable}, default 'default' - Controls the SQL insertion clause used. + Controls the SQL insertion clause used: - * `'default'`: Uses standard SQL `INSERT` clause (one per row). - * `'multi'`: Pass multiple values in a single `INSERT` clause. - * callable: with signature `(pd_table, conn, keys, data_iter)`. + * 'default': Uses standard SQL ``INSERT`` clause (one per row). + * 'multi': Pass multiple values in a single ``INSERT`` clause. + * callable with signature ``(pd_table, conn, keys, data_iter)``. - Details and a sample callable implementation on + Details and a sample callable implementation can be found in the section :ref:`insert method `. .. versionadded:: 0.24.0 @@ -1149,13 +1149,13 @@ def to_sql(self, frame, name, if_exists='fail', index=True, be a SQLAlchemy type. If all columns are of the same type, one single value can be used. method : {'default', 'multi', callable}, default 'default' - Controls the SQL insertion clause used. + Controls the SQL insertion clause used: - * `'default'`: Uses standard SQL `INSERT` clause (one per row). - * `'multi'`: Pass multiple values in a single `INSERT` clause. - * callable: with signature `(pd_table, conn, keys, data_iter)`. + * 'default': Uses standard SQL ``INSERT`` clause (one per row). + * 'multi': Pass multiple values in a single ``INSERT`` clause. + * callable with signature ``(pd_table, conn, keys, data_iter)``. - Details and a sample callable implementation on + Details and a sample callable implementation can be found in the section :ref:`insert method `. .. versionadded:: 0.24.0 @@ -1515,13 +1515,13 @@ def to_sql(self, frame, name, if_exists='fail', index=True, be a string. If all columns are of the same type, one single value can be used. method : {'default', 'multi', callable}, default 'default' - Controls the SQL insertion clause used. + Controls the SQL insertion clause used: - * `'default'`: Uses standard SQL `INSERT` clause (one per row). - * `'multi'`: Pass multiple values in a single `INSERT` clause. - * callable: with signature `(pd_table, conn, keys, data_iter)`. + * 'default': Uses standard SQL ``INSERT`` clause (one per row). + * 'multi': Pass multiple values in a single ``INSERT`` clause. + * callable with signature ``(pd_table, conn, keys, data_iter)``. - Details and a sample callable implementation on + Details and a sample callable implementation can be found in the section :ref:`insert method `. .. versionadded:: 0.24.0 From b49792b9903c3e17fbb1ba2e57e6702160f73f5b Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Wed, 24 Oct 2018 11:06:24 +0200 Subject: [PATCH 06/16] additional doc clean-up --- pandas/core/generic.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pandas/core/generic.py b/pandas/core/generic.py index e302a0dd85783..560b2c80ecff7 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -2295,13 +2295,13 @@ def to_sql(self, name, con, schema=None, if_exists='fail', index=True, names and the values should be the SQLAlchemy types or strings for the sqlite3 legacy mode. method : {'default', 'multi', callable}, default 'default' - Controls the SQL insertion clause used. + Controls the SQL insertion clause used: - * `'default'`: Uses standard SQL `INSERT` clause (one per row). - * `'multi'`: Pass multiple values in a single `INSERT` clause. - * callable: with signature `(pd_table, conn, keys, data_iter)`. + * 'default': Uses standard SQL ``INSERT`` clause (one per row). + * 'multi': Pass multiple values in a single ``INSERT`` clause. + * callable with signature ``(pd_table, conn, keys, data_iter)``. - Details and a sample callable implementation on + Details and a sample callable implementation can be found in the section :ref:`insert method `. .. versionadded:: 0.24.0 From 7bdf6f38789975ff5daef831ceceadf11cc00966 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Tue, 25 Dec 2018 16:10:54 -0800 Subject: [PATCH 07/16] use dict(zip()) directly --- pandas/io/sql.py | 4 ++-- pandas/tests/io/test_sql.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pandas/io/sql.py b/pandas/io/sql.py index c13d177789b6c..ac18da5a8e725 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -591,7 +591,7 @@ def _execute_insert(self, conn, keys, data_iter): data_iter : generator of list Each item contains a list of values to be inserted """ - data = [{k: v for k, v in zip(keys, row)} for row in data_iter] + data = [dict(zip(keys, row)) for row in data_iter] conn.execute(self.table.insert(), data) def _execute_insert_multi(self, conn, keys, data_iter): @@ -601,7 +601,7 @@ def _execute_insert_multi(self, conn, keys, data_iter): and tables containing a few columns but performance degrades quickly with increase of columns. """ - data = [{k: v for k, v in zip(keys, row)} for row in data_iter] + data = [dict(zip(keys, row)) for row in data_iter] conn.execute(self.table.insert(data)) def insert_data(self): diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 7a16e4b659ff6..b19485010bfc7 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -443,7 +443,7 @@ def _to_sql_method_callable(self): def sample(pd_table, conn, keys, data_iter): check.append(1) - data = [{k: v for k, v in zip(keys, row)} for row in data_iter] + data = [dict(zip(keys, row)) for row in data_iter] conn.execute(pd_table.table.insert(), data) self.drop_table('test_frame1') From 44f321a53742dfb6d0a510cc180e021975725685 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Tue, 25 Dec 2018 23:03:48 -0800 Subject: [PATCH 08/16] clean up merge --- doc/source/io.rst | 63 ++++++++++++++++----------------- doc/source/whatsnew/v0.24.0.rst | 6 +--- 2 files changed, 31 insertions(+), 38 deletions(-) diff --git a/doc/source/io.rst b/doc/source/io.rst index 653fe063ef8af..d10fc32a50477 100644 --- a/doc/source/io.rst +++ b/doc/source/io.rst @@ -4959,7 +4959,36 @@ default ``Text`` type for string columns: Because of this, reading the database table back in does **not** generate a categorical. -<<<<<<< HEAD +.. _io.sql_datetime_data: + +Datetime data types +''''''''''''''''''' + +Using SQLAlchemy, :func:`~pandas.DataFrame.to_sql` is capable of writing +datetime data that is timezone naive or timezone aware. However, the resulting +data stored in the database ultimately depends on the supported data type +for datetime data of the database system being used. + +The following table lists supported data types for datetime data for some +common databases. Other database dialects may have different data types for +datetime data. + +=========== ============================================= =================== +Database SQL Datetime Types Timezone Support +=========== ============================================= =================== +SQLite ``TEXT`` No +MySQL ``TIMESTAMP`` or ``DATETIME`` No +PostgreSQL ``TIMESTAMP`` or ``TIMESTAMP WITH TIME ZONE`` Yes +=========== ============================================= =================== + +When writing timezone aware data to databases that do not support timezones, +the data will be written as timezone naive timestamps that are in local time +with respect to the timezone. + +:func:`~pandas.read_sql_table` is also capable of reading datetime data that is +timezone aware or naive. When reading ``TIMESTAMP WITH TIME ZONE`` types, pandas +will convert the data to UTC. + .. _io.sql.method: Insertion Method @@ -5008,38 +5037,6 @@ Example of a callable using PostgreSQL `COPY clause table_name, columns) cur.copy_expert(sql=sql, file=s_buf) -======= -.. _io.sql_datetime_data: - -Datetime data types -''''''''''''''''''' - -Using SQLAlchemy, :func:`~pandas.DataFrame.to_sql` is capable of writing -datetime data that is timezone naive or timezone aware. However, the resulting -data stored in the database ultimately depends on the supported data type -for datetime data of the database system being used. - -The following table lists supported data types for datetime data for some -common databases. Other database dialects may have different data types for -datetime data. - -=========== ============================================= =================== -Database SQL Datetime Types Timezone Support -=========== ============================================= =================== -SQLite ``TEXT`` No -MySQL ``TIMESTAMP`` or ``DATETIME`` No -PostgreSQL ``TIMESTAMP`` or ``TIMESTAMP WITH TIME ZONE`` Yes -=========== ============================================= =================== - -When writing timezone aware data to databases that do not support timezones, -the data will be written as timezone naive timestamps that are in local time -with respect to the timezone. - -:func:`~pandas.read_sql_table` is also capable of reading datetime data that is -timezone aware or naive. When reading ``TIMESTAMP WITH TIME ZONE`` types, pandas -will convert the data to UTC. ->>>>>>> upstream/master - Reading Tables '''''''''''''' diff --git a/doc/source/whatsnew/v0.24.0.rst b/doc/source/whatsnew/v0.24.0.rst index 81234a94bcbed..8bba8c41807a4 100644 --- a/doc/source/whatsnew/v0.24.0.rst +++ b/doc/source/whatsnew/v0.24.0.rst @@ -331,12 +331,7 @@ Other Enhancements - :func:`to_datetime` now supports the ``%Z`` and ``%z`` directive when passed into ``format`` (:issue:`13486`) - :func:`Series.mode` and :func:`DataFrame.mode` now support the ``dropna`` parameter which can be used to specify whether ``NaN``/``NaT`` values should be considered (:issue:`17534`) -<<<<<<< HEAD:doc/source/whatsnew/v0.24.0.txt -- :func:`to_csv` now supports ``compression`` keyword when a file handle is passed. (:issue:`21227`) -- :func:`~pandas.DataFrame.to_sql` add parameter ``method`` to control SQL insertion clause (:8953:) -======= - :func:`DataFrame.to_csv` and :func:`Series.to_csv` now support the ``compression`` keyword when a file handle is passed. (:issue:`21227`) ->>>>>>> upstream/master:doc/source/whatsnew/v0.24.0.rst - :meth:`Index.droplevel` is now implemented also for flat indexes, for compatibility with :class:`MultiIndex` (:issue:`21115`) - :meth:`Series.droplevel` and :meth:`DataFrame.droplevel` are now implemented (:issue:`20342`) - Added support for reading from/writing to Google Cloud Storage via the ``gcsfs`` library (:issue:`19454`, :issue:`23094`) @@ -378,6 +373,7 @@ Other Enhancements - :meth:`DataFrame.between_time` and :meth:`DataFrame.at_time` have gained the an ``axis`` parameter (:issue:`8839`) - The ``scatter_matrix``, ``andrews_curves``, ``parallel_coordinates``, ``lag_plot``, ``autocorrelation_plot``, ``bootstrap_plot``, and ``radviz`` plots from the ``pandas.plotting`` module are now accessible from calling :meth:`DataFrame.plot` (:issue:`11978`) - :class:`IntervalIndex` has gained the :attr:`~IntervalIndex.is_overlapping` attribute to indicate if the ``IntervalIndex`` contains any overlapping intervals (:issue:`23309`) +- :func:`pandas.DataFrame.to_sql` has gained the ``method`` argument to control SQL insertion clause (:issue:`8953`) .. _whatsnew_0240.api_breaking: From d5ccabfe17680ea2005e0e97962ef9c717c66722 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Wed, 26 Dec 2018 10:20:33 -0800 Subject: [PATCH 09/16] default --> None --- doc/source/io.rst | 2 +- pandas/core/generic.py | 6 +++--- pandas/io/sql.py | 22 +++++++++++----------- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/doc/source/io.rst b/doc/source/io.rst index d10fc32a50477..fa6a8b1d01530 100644 --- a/doc/source/io.rst +++ b/doc/source/io.rst @@ -4999,7 +4999,7 @@ Insertion Method The parameter ``method`` controls the SQL insertion clause used. Possible values are: -- ``'default'``: Uses standard SQL ``INSERT`` clause (one per row). +- ``None``: Uses standard SQL ``INSERT`` clause (one per row). - ``'multi'``: Pass multiple values in a single ``INSERT`` clause. It uses a *special* SQL syntax not supported by all backends. This usually provides better performance for analytic databases diff --git a/pandas/core/generic.py b/pandas/core/generic.py index ee29e4c17a1e1..3c28fef024b79 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -2386,7 +2386,7 @@ def to_msgpack(self, path_or_buf=None, encoding='utf-8', **kwargs): **kwargs) def to_sql(self, name, con, schema=None, if_exists='fail', index=True, - index_label=None, chunksize=None, dtype=None, method='default'): + index_label=None, chunksize=None, dtype=None, method=None): """ Write records stored in a DataFrame to a SQL database. @@ -2424,10 +2424,10 @@ def to_sql(self, name, con, schema=None, if_exists='fail', index=True, Specifying the datatype for columns. The keys should be the column names and the values should be the SQLAlchemy types or strings for the sqlite3 legacy mode. - method : {'default', 'multi', callable}, default 'default' + method : {None, 'multi', callable}, default None Controls the SQL insertion clause used: - * 'default': Uses standard SQL ``INSERT`` clause (one per row). + * None : Uses standard SQL ``INSERT`` clause (one per row). * 'multi': Pass multiple values in a single ``INSERT`` clause. * callable with signature ``(pd_table, conn, keys, data_iter)``. diff --git a/pandas/io/sql.py b/pandas/io/sql.py index ac18da5a8e725..6093c6c3fd0fc 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -396,7 +396,7 @@ def read_sql(sql, con, index_col=None, coerce_float=True, params=None, def to_sql(frame, name, con, schema=None, if_exists='fail', index=True, - index_label=None, chunksize=None, dtype=None, method='default'): + index_label=None, chunksize=None, dtype=None, method=None): """ Write records stored in a DataFrame to a SQL database. @@ -430,10 +430,10 @@ def to_sql(frame, name, con, schema=None, if_exists='fail', index=True, Optional specifying the datatype for columns. The SQL type should be a SQLAlchemy type, or a string for sqlite3 fallback connection. If all columns are of the same type, one single value can be used. - method : {'default', 'multi', callable}, default 'default' + method : {None, 'multi', callable}, default None Controls the SQL insertion clause used: - - 'default': Uses standard SQL ``INSERT`` clause (one per row). + - None : Uses standard SQL ``INSERT`` clause (one per row). - 'multi': Pass multiple values in a single ``INSERT`` clause. - callable with signature ``(pd_table, conn, keys, data_iter)``. @@ -645,10 +645,10 @@ def insert_data(self): return column_names, data_list - def insert(self, chunksize=None, method='default'): + def insert(self, chunksize=None, method=None): # set insert method - if method == 'default': + if method is None: exec_insert = self._execute_insert elif method == 'multi': exec_insert = self._execute_insert_multi @@ -1126,7 +1126,7 @@ def read_query(self, sql, index_col=None, coerce_float=True, def to_sql(self, frame, name, if_exists='fail', index=True, index_label=None, schema=None, chunksize=None, dtype=None, - method='default'): + method=None): """ Write records stored in a DataFrame to a SQL database. @@ -1156,10 +1156,10 @@ def to_sql(self, frame, name, if_exists='fail', index=True, Optional specifying the datatype for columns. The SQL type should be a SQLAlchemy type. If all columns are of the same type, one single value can be used. - method : {'default', 'multi', callable}, default 'default' + method : {None', 'multi', callable}, default None Controls the SQL insertion clause used: - * 'default': Uses standard SQL ``INSERT`` clause (one per row). + * None : Uses standard SQL ``INSERT`` clause (one per row). * 'multi': Pass multiple values in a single ``INSERT`` clause. * callable with signature ``(pd_table, conn, keys, data_iter)``. @@ -1494,7 +1494,7 @@ def _fetchall_as_list(self, cur): def to_sql(self, frame, name, if_exists='fail', index=True, index_label=None, schema=None, chunksize=None, dtype=None, - method='default'): + method=None): """ Write records stored in a DataFrame to a SQL database. @@ -1523,10 +1523,10 @@ def to_sql(self, frame, name, if_exists='fail', index=True, Optional specifying the datatype for columns. The SQL type should be a string. If all columns are of the same type, one single value can be used. - method : {'default', 'multi', callable}, default 'default' + method : {None, 'multi', callable}, default None Controls the SQL insertion clause used: - * 'default': Uses standard SQL ``INSERT`` clause (one per row). + * None : Uses standard SQL ``INSERT`` clause (one per row). * 'multi': Pass multiple values in a single ``INSERT`` clause. * callable with signature ``(pd_table, conn, keys, data_iter)``. From 643e9bf1cae4dd05fb328611ec839ff65c68b502 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Wed, 26 Dec 2018 10:56:27 -0800 Subject: [PATCH 10/16] Remove stray default --- pandas/tests/io/test_sql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index b19485010bfc7..897108d1aaf7b 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -375,7 +375,7 @@ def _read_sql_iris_named_parameter(self): iris_frame = self.pandasSQL.read_query(query, params=params) self._check_iris_loaded_frame(iris_frame) - def _to_sql(self, method='default'): + def _to_sql(self): self.drop_table('test_frame1') self.pandasSQL.to_sql(self.test_frame1, 'test_frame1', method=method) From 6fc6a2672d205bb934da542a08edf1a485c22a7e Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Wed, 26 Dec 2018 10:57:40 -0800 Subject: [PATCH 11/16] Remove method kwarg --- pandas/tests/io/test_sql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 897108d1aaf7b..5417dba658aac 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -378,7 +378,7 @@ def _read_sql_iris_named_parameter(self): def _to_sql(self): self.drop_table('test_frame1') - self.pandasSQL.to_sql(self.test_frame1, 'test_frame1', method=method) + self.pandasSQL.to_sql(self.test_frame1, 'test_frame1') assert self.pandasSQL.has_table('test_frame1') num_entries = len(self.test_frame1) From 87730f313d0aaa6a70a404c7cb69d70f324816ed Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Wed, 26 Dec 2018 11:29:44 -0800 Subject: [PATCH 12/16] change default to None --- pandas/tests/io/test_sql.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 5417dba658aac..261bdae072aec 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -375,10 +375,10 @@ def _read_sql_iris_named_parameter(self): iris_frame = self.pandasSQL.read_query(query, params=params) self._check_iris_loaded_frame(iris_frame) - def _to_sql(self): + def _to_sql(self, method=None): self.drop_table('test_frame1') - self.pandasSQL.to_sql(self.test_frame1, 'test_frame1') + self.pandasSQL.to_sql(self.test_frame1, 'test_frame1', method=method) assert self.pandasSQL.has_table('test_frame1') num_entries = len(self.test_frame1) From f36710b63dd1b35cc4fd950d33cdca0982960e04 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Thu, 27 Dec 2018 13:19:48 -0800 Subject: [PATCH 13/16] test copy insert snippit --- pandas/tests/io/test_sql.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 261bdae072aec..a1dc4fbb7f612 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -1959,6 +1959,35 @@ def test_schema_support(self): res2 = pdsql.read_table('test_schema_other2') tm.assert_frame_equal(res1, res2) + def test_copy_from_callable_insertion_method(self): + # GH 8953 + # Example in io.rst found under _io.sql.method + # not available in sqlite, mysql + def psql_insert_copy(table, conn, keys, data_iter): + # gets a DBAPI connection that can provide a cursor + dbapi_conn = conn.connection + with dbapi_conn.cursor() as cur: + s_buf = compat.StringIO() + writer = csv.writer(s_buf) + writer.writerows(data_iter) + s_buf.seek(0) + + columns = ', '.join('"{}"'.format(k) for k in keys) + if table.schema: + table_name = '{}.{}'.format(table.schema, table.name) + else: + table_name = table.name + + sql_query = 'COPY {} ({}) FROM STDIN WITH CSV'.format( + table_name, columns) + cur.copy_expert(sql=sql_query, file=s_buf) + + expected = DataFrame({'col1': [1, 2], 'col2': [0.1, 0.2], + 'col3': ['a', 'n']}) + expected.to_sql('test_copy_insert', self.conn, method=psql_insert_copy) + result = sql.read_sql_table('test_copy_insert', self.conn) + tm.assert_frame_equal(result, expected) + @pytest.mark.single class TestMySQLAlchemy(_TestMySQLAlchemy, _TestSQLAlchemy): From 19f9dfa91c438fbaaa07a279ec793c5fc0acfc0c Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Thu, 27 Dec 2018 14:08:01 -0800 Subject: [PATCH 14/16] print debug --- pandas/tests/io/test_sql.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index a1dc4fbb7f612..327b4e14e9997 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -1986,6 +1986,8 @@ def psql_insert_copy(table, conn, keys, data_iter): 'col3': ['a', 'n']}) expected.to_sql('test_copy_insert', self.conn, method=psql_insert_copy) result = sql.read_sql_table('test_copy_insert', self.conn) + print(result) + print(expected) tm.assert_frame_equal(result, expected) From c0bf4575d02961302190031292648d16b66acb2b Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Thu, 27 Dec 2018 14:41:14 -0800 Subject: [PATCH 15/16] index=False --- pandas/tests/io/test_sql.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 327b4e14e9997..c346103a70c98 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -1984,10 +1984,9 @@ def psql_insert_copy(table, conn, keys, data_iter): expected = DataFrame({'col1': [1, 2], 'col2': [0.1, 0.2], 'col3': ['a', 'n']}) - expected.to_sql('test_copy_insert', self.conn, method=psql_insert_copy) + expected.to_sql('test_copy_insert', self.conn, index=False, + method=psql_insert_copy) result = sql.read_sql_table('test_copy_insert', self.conn) - print(result) - print(expected) tm.assert_frame_equal(result, expected) From 19ce379e83e4eba68d24d194083b512557ffad1d Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Thu, 27 Dec 2018 15:50:26 -0800 Subject: [PATCH 16/16] Add reference to documentation --- doc/source/whatsnew/v0.24.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v0.24.0.rst b/doc/source/whatsnew/v0.24.0.rst index 8bba8c41807a4..6cb08afadec31 100644 --- a/doc/source/whatsnew/v0.24.0.rst +++ b/doc/source/whatsnew/v0.24.0.rst @@ -373,7 +373,7 @@ Other Enhancements - :meth:`DataFrame.between_time` and :meth:`DataFrame.at_time` have gained the an ``axis`` parameter (:issue:`8839`) - The ``scatter_matrix``, ``andrews_curves``, ``parallel_coordinates``, ``lag_plot``, ``autocorrelation_plot``, ``bootstrap_plot``, and ``radviz`` plots from the ``pandas.plotting`` module are now accessible from calling :meth:`DataFrame.plot` (:issue:`11978`) - :class:`IntervalIndex` has gained the :attr:`~IntervalIndex.is_overlapping` attribute to indicate if the ``IntervalIndex`` contains any overlapping intervals (:issue:`23309`) -- :func:`pandas.DataFrame.to_sql` has gained the ``method`` argument to control SQL insertion clause (:issue:`8953`) +- :func:`pandas.DataFrame.to_sql` has gained the ``method`` argument to control SQL insertion clause. See the :ref:`insertion method ` section in the documentation. (:issue:`8953`) .. _whatsnew_0240.api_breaking: