From 91b934cb243f92b0b54ffff867185b4e026956b1 Mon Sep 17 00:00:00 2001 From: lzieglerpenn Date: Mon, 5 Dec 2022 14:39:30 -0500 Subject: [PATCH 1/9] add functions and docs for to_sql if exists --- pandas/io/sql.py | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/pandas/io/sql.py b/pandas/io/sql.py index e3510f71bd0cd..47ef37ca6ba97 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -604,7 +604,7 @@ def to_sql( name: str, con, schema: str | None = None, - if_exists: Literal["fail", "replace", "append"] = "fail", + if_exists: Literal["fail", "replace", "append", "truncate"] = "fail", index: bool = True, index_label: IndexLabel = None, chunksize: int | None = None, @@ -629,10 +629,11 @@ def to_sql( schema : str, optional Name of SQL schema in database to write to (if database flavor supports this). If None, use default schema (default). - if_exists : {'fail', 'replace', 'append'}, default 'fail' + if_exists : {'fail', 'replace', 'append', 'truncate}, default 'fail' - fail: If table exists, do nothing. - replace: If table exists, drop it, recreate it, and insert data. - append: If table exists, insert data. Create if does not exist. + - truncate: If table exists, truncate it, then insert data. index : bool, default True Write DataFrame index as a column. index_label : str or sequence, optional @@ -682,7 +683,7 @@ def to_sql( `sqlite3 `__ or `SQLAlchemy `__ """ # noqa:E501 - if if_exists not in ("fail", "replace", "append"): + if if_exists not in ("fail", "replace", "append", "truncate"): raise ValueError(f"'{if_exists}' is not valid for if_exists") if isinstance(frame, Series): @@ -854,6 +855,9 @@ def create(self) -> None: if self.if_exists == "replace": self.pd_sql.drop_table(self.name, self.schema) self._execute_create() + elif self.if_exists == "truncate": + self.pd_sql.trunc_table(self.name, self.schema) + self._execute_create() elif self.if_exists == "append": pass else: @@ -1311,7 +1315,7 @@ def to_sql( self, frame, name, - if_exists: Literal["fail", "replace", "append"] = "fail", + if_exists: Literal["fail", "replace", "append", "truncate"] = "fail", index: bool = True, index_label=None, schema=None, @@ -1642,7 +1646,7 @@ def prep_table( self, frame, name, - if_exists: Literal["fail", "replace", "append"] = "fail", + if_exists: Literal["fail", "replace", "append", "truncate"] = "fail", index: bool | str | list[str] | None = True, index_label=None, schema=None, @@ -1718,7 +1722,7 @@ def to_sql( self, frame, name: str, - if_exists: Literal["fail", "replace", "append"] = "fail", + if_exists: Literal["fail", "replace", "append", "truncate"] = "fail", index: bool = True, index_label=None, schema: str | None = None, @@ -1736,10 +1740,11 @@ def to_sql( frame : DataFrame name : string Name of SQL table. - if_exists : {'fail', 'replace', 'append'}, default 'fail' + if_exists : {'fail', 'replace', 'append', 'truncate'}, default 'fail' - fail: If table exists, do nothing. - replace: If table exists, drop it, recreate it, and insert data. - append: If table exists, insert data. Create if does not exist. + - truncate: If table exists, truncate it, and insert data. index : boolean, default True Write DataFrame index as a column. index_label : string or sequence, default None @@ -1833,6 +1838,15 @@ def drop_table(self, table_name: str, schema: str | None = None) -> None: self.get_table(table_name, schema).drop(bind=self.con) self.meta.clear() + + def trunc_table(self, table_name: str, schema: str | None = None) -> None: + schema = schema or self.meta.schema + if self.has_table(table_name, schema): + self.meta.reflect(bind=self.con, only=[table_name], schema=schema) + # self.execute(f"TRUNCATE TABLE {schema}.{table_name};") -- true truncate + self.get_table(table_name, schema).delete(bind=self.con) + self.meta.clear() + def _create_sql_schema( self, frame: DataFrame, @@ -2181,10 +2195,11 @@ def to_sql( frame: DataFrame name: string Name of SQL table. - if_exists: {'fail', 'replace', 'append'}, default 'fail' + if_exists: {'fail', 'replace', 'append', 'truncate}, default 'fail' fail: If table exists, do nothing. replace: If table exists, drop it, recreate it, and insert data. append: If table exists, insert data. Create if it does not exist. + truncate: If table exists, truncate it, then insert data. index : bool, default True Write DataFrame index as a column index_label : string or sequence, default None @@ -2253,6 +2268,10 @@ def drop_table(self, name: str, schema: str | None = None) -> None: drop_sql = f"DROP TABLE {_get_valid_sqlite_name(name)}" self.execute(drop_sql) + def trunc_table(self, name: str, schema: str | None = None) -> None: + trunc_sql = f"TRUNCATE TABLE {_get_valid_sqlite_name(name)}" + self.execute(trunc_sql) + def _create_sql_schema( self, frame, From c9555ad49a3831969e48f9656c064247b1db675d Mon Sep 17 00:00:00 2001 From: lzieglerpenn Date: Mon, 5 Dec 2022 14:57:00 -0500 Subject: [PATCH 2/9] add test for to_sql truncate --- pandas/tests/io/test_sql.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 394fceb69b788..061c40f469797 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -902,6 +902,17 @@ def test_to_sql_replace(self, test_frame1): assert num_rows == num_entries + def test_to_sql_truncate(self, test_frame1): + sql.to_sql(test_frame1, "test_frame3", self.conn, if_exists="fail") + # Add to table again + sql.to_sql(test_frame1, "test_frame3", self.conn, if_exists="truncate") + assert sql.has_table("test_frame3", self.conn) + + num_entries = len(test_frame1) + num_rows = count_rows(self.conn, "test_frame3") + + assert num_rows == num_entries + def test_to_sql_append(self, test_frame1): assert sql.to_sql(test_frame1, "test_frame4", self.conn, if_exists="fail") == 4 From 9ad13531f2ddf9878eb08a2b328223486af224eb Mon Sep 17 00:00:00 2001 From: lzieglerpenn Date: Tue, 6 Dec 2022 09:32:26 -0500 Subject: [PATCH 3/9] update formating --- pandas/io/sql.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pandas/io/sql.py b/pandas/io/sql.py index 47ef37ca6ba97..ec3765ae607d2 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -1838,7 +1838,6 @@ def drop_table(self, table_name: str, schema: str | None = None) -> None: self.get_table(table_name, schema).drop(bind=self.con) self.meta.clear() - def trunc_table(self, table_name: str, schema: str | None = None) -> None: schema = schema or self.meta.schema if self.has_table(table_name, schema): From b1ae19a4b4ea37708a25c77b65aa4272e1cc4788 Mon Sep 17 00:00:00 2001 From: lzieglerpenn Date: Tue, 6 Dec 2022 12:40:34 -0500 Subject: [PATCH 4/9] better trunc function and doc whats new --- doc/source/whatsnew/v1.5.3.rst | 2 +- pandas/io/sql.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/source/whatsnew/v1.5.3.rst b/doc/source/whatsnew/v1.5.3.rst index b6c1c857717c7..a79d518d24ead 100644 --- a/doc/source/whatsnew/v1.5.3.rst +++ b/doc/source/whatsnew/v1.5.3.rst @@ -32,7 +32,7 @@ Bug fixes Other ~~~~~ -- +- Added ``"truncate"`` option to ``if_exists`` argument in :meth:`DataFrame.to_sql` to truncate the existing table (:issue:`37210`) - .. --------------------------------------------------------------------------- diff --git a/pandas/io/sql.py b/pandas/io/sql.py index ec3765ae607d2..d194dc7132042 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -857,7 +857,6 @@ def create(self) -> None: self._execute_create() elif self.if_exists == "truncate": self.pd_sql.trunc_table(self.name, self.schema) - self._execute_create() elif self.if_exists == "append": pass else: @@ -1842,8 +1841,9 @@ def trunc_table(self, table_name: str, schema: str | None = None) -> None: schema = schema or self.meta.schema if self.has_table(table_name, schema): self.meta.reflect(bind=self.con, only=[table_name], schema=schema) - # self.execute(f"TRUNCATE TABLE {schema}.{table_name};") -- true truncate - self.get_table(table_name, schema).delete(bind=self.con) + if schema: + schema=schema+'.' + self.execute(f"DELETE FROM {schema or ''}{table_name}") self.meta.clear() def _create_sql_schema( From e11b69bdc40d487a8cf06218e5a22ebf13bea68e Mon Sep 17 00:00:00 2001 From: lzieglerpenn Date: Tue, 6 Dec 2022 12:41:43 -0500 Subject: [PATCH 5/9] precommit whitespace removal --- pandas/io/sql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/io/sql.py b/pandas/io/sql.py index d194dc7132042..f95d3a33bbe45 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -1842,7 +1842,7 @@ def trunc_table(self, table_name: str, schema: str | None = None) -> None: if self.has_table(table_name, schema): self.meta.reflect(bind=self.con, only=[table_name], schema=schema) if schema: - schema=schema+'.' + schema = schema + "." self.execute(f"DELETE FROM {schema or ''}{table_name}") self.meta.clear() From 0db136d9d9005c2e054051a7c5d1c7fefa4602b9 Mon Sep 17 00:00:00 2001 From: lzieglerpenn Date: Sat, 17 Dec 2022 16:41:57 -0500 Subject: [PATCH 6/9] added new tests & requested changes --- doc/source/whatsnew/v1.5.3.rst | 2 +- doc/source/whatsnew/v2.0.0.rst | 2 +- pandas/io/sql.py | 5 +++-- pandas/tests/io/test_sql.py | 14 ++++++++++++++ 4 files changed, 19 insertions(+), 4 deletions(-) diff --git a/doc/source/whatsnew/v1.5.3.rst b/doc/source/whatsnew/v1.5.3.rst index a79d518d24ead..ffffd8ee86987 100644 --- a/doc/source/whatsnew/v1.5.3.rst +++ b/doc/source/whatsnew/v1.5.3.rst @@ -32,7 +32,7 @@ Bug fixes Other ~~~~~ -- Added ``"truncate"`` option to ``if_exists`` argument in :meth:`DataFrame.to_sql` to truncate the existing table (:issue:`37210`) +- - .. --------------------------------------------------------------------------- diff --git a/doc/source/whatsnew/v2.0.0.rst b/doc/source/whatsnew/v2.0.0.rst index 5b57baa9ec39a..515b86ea24bf5 100644 --- a/doc/source/whatsnew/v2.0.0.rst +++ b/doc/source/whatsnew/v2.0.0.rst @@ -83,7 +83,7 @@ Other enhancements - :func:`timedelta_range` now supports a ``unit`` keyword ("s", "ms", "us", or "ns") to specify the desired resolution of the output index (:issue:`49824`) - :meth:`DataFrame.to_json` now supports a ``mode`` keyword with supported inputs 'w' and 'a'. Defaulting to 'w', 'a' can be used when lines=True and orient='records' to append record oriented json lines to an existing json file. (:issue:`35849`) - Added ``name`` parameter to :meth:`IntervalIndex.from_breaks`, :meth:`IntervalIndex.from_arrays` and :meth:`IntervalIndex.from_tuples` (:issue:`48911`) -- +- Added ``"truncate"`` option to ``if_exists`` argument in :meth:`DataFrame.to_sql` to truncate the existing table (:issue:`37210`) .. --------------------------------------------------------------------------- .. _whatsnew_200.notable_bug_fixes: diff --git a/pandas/io/sql.py b/pandas/io/sql.py index f95d3a33bbe45..99038e64fcc93 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -1842,8 +1842,9 @@ def trunc_table(self, table_name: str, schema: str | None = None) -> None: if self.has_table(table_name, schema): self.meta.reflect(bind=self.con, only=[table_name], schema=schema) if schema: - schema = schema + "." - self.execute(f"DELETE FROM {schema or ''}{table_name}") + self.execute(f"TRUNCATE FROM {schema}.{table_name}") + else: + self.execute(f"DELETE FROM {table_name}") self.meta.clear() def _create_sql_schema( diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 061c40f469797..d1cd792dc8d08 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -913,6 +913,20 @@ def test_to_sql_truncate(self, test_frame1): assert num_rows == num_entries + def test_to_sql_truncate_no_table(self, test_frame1): + # creates new table if table doesn't exist + sql.to_sql(test_frame1, "test_frame_new", self.conn, if_exists="truncate") + assert sql.has_table("test_frame_new") + + def test_to_sql_truncate_new_columns(self, test_frame1, test_frame3): + sql.to_sql(test_frame3, "test_frame3", self.conn, if_exists='fail') + # truncate and attempt to add more columns + msg = "table test_frame3 has no column named C" + with pytest.raises(Exception, match=msg): + sql.to_sql(test_frame1, "test_frame3", self.conn, if_exists='truncate') + + + def test_to_sql_append(self, test_frame1): assert sql.to_sql(test_frame1, "test_frame4", self.conn, if_exists="fail") == 4 From 9ec6ecdded8ef4a3cea27ce9023e8c8fdb21e330 Mon Sep 17 00:00:00 2001 From: lzieglerpenn Date: Thu, 22 Dec 2022 12:04:23 -0500 Subject: [PATCH 7/9] remove db dependent if statement --- pandas/io/sql.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/pandas/io/sql.py b/pandas/io/sql.py index 99038e64fcc93..7fbfba9cd74dd 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -1841,10 +1841,7 @@ def trunc_table(self, table_name: str, schema: str | None = None) -> None: schema = schema or self.meta.schema if self.has_table(table_name, schema): self.meta.reflect(bind=self.con, only=[table_name], schema=schema) - if schema: - self.execute(f"TRUNCATE FROM {schema}.{table_name}") - else: - self.execute(f"DELETE FROM {table_name}") + self.execute(f"TRUNCATE FROM {schema}.{table_name}") self.meta.clear() def _create_sql_schema( @@ -2269,7 +2266,7 @@ def drop_table(self, name: str, schema: str | None = None) -> None: self.execute(drop_sql) def trunc_table(self, name: str, schema: str | None = None) -> None: - trunc_sql = f"TRUNCATE TABLE {_get_valid_sqlite_name(name)}" + trunc_sql = f"DELETE FROM {_get_valid_sqlite_name(name)}" self.execute(trunc_sql) def _create_sql_schema( From aa75b316a4bf1e6ba1e4f38a400ef8789eef2a75 Mon Sep 17 00:00:00 2001 From: lzieglerpenn Date: Thu, 22 Dec 2022 12:11:34 -0500 Subject: [PATCH 8/9] TRUNCATE TABLE Typo --- pandas/io/sql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/io/sql.py b/pandas/io/sql.py index 7fbfba9cd74dd..4bc0bacfff2c4 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -1841,7 +1841,7 @@ def trunc_table(self, table_name: str, schema: str | None = None) -> None: schema = schema or self.meta.schema if self.has_table(table_name, schema): self.meta.reflect(bind=self.con, only=[table_name], schema=schema) - self.execute(f"TRUNCATE FROM {schema}.{table_name}") + self.execute(f"TRUNCATE TABLE {schema}.{table_name}") self.meta.clear() def _create_sql_schema( From bc3e3a424675ec2606543fbe7cd250ffc90442f8 Mon Sep 17 00:00:00 2001 From: lzieglerpenn Date: Thu, 22 Dec 2022 13:59:16 -0500 Subject: [PATCH 9/9] add NotImplementedError --- pandas/io/sql.py | 3 +-- pandas/tests/io/test_sql.py | 15 ++++++++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/pandas/io/sql.py b/pandas/io/sql.py index 4bc0bacfff2c4..6a307340eb31b 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -2266,8 +2266,7 @@ def drop_table(self, name: str, schema: str | None = None) -> None: self.execute(drop_sql) def trunc_table(self, name: str, schema: str | None = None) -> None: - trunc_sql = f"DELETE FROM {_get_valid_sqlite_name(name)}" - self.execute(trunc_sql) + raise NotImplementedError("TRUNCATE not implemented on database") def _create_sql_schema( self, diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index d1cd792dc8d08..58518d9de6273 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -917,15 +917,20 @@ def test_to_sql_truncate_no_table(self, test_frame1): # creates new table if table doesn't exist sql.to_sql(test_frame1, "test_frame_new", self.conn, if_exists="truncate") assert sql.has_table("test_frame_new") - + def test_to_sql_truncate_new_columns(self, test_frame1, test_frame3): - sql.to_sql(test_frame3, "test_frame3", self.conn, if_exists='fail') + sql.to_sql(test_frame3, "test_frame3", self.conn, if_exists="fail") # truncate and attempt to add more columns msg = "table test_frame3 has no column named C" with pytest.raises(Exception, match=msg): - sql.to_sql(test_frame1, "test_frame3", self.conn, if_exists='truncate') - - + sql.to_sql(test_frame1, "test_frame3", self.conn, if_exists="truncate") + + def test_sqlite_truncate_raises(self, test_frame1): + msg = "TRUNCATE not implemented on database" + with pytest.raises(NotImplementedError, match=msg): + sql.to_sql( + test_frame1, "test_frame3", self.conn, if_exists="truncate" + ) def test_to_sql_append(self, test_frame1): assert sql.to_sql(test_frame1, "test_frame4", self.conn, if_exists="fail") == 4