diff --git a/doc/source/whatsnew/v2.1.0.rst b/doc/source/whatsnew/v2.1.0.rst index e1ac9e3309de7..5ebbe1abeb7f7 100644 --- a/doc/source/whatsnew/v2.1.0.rst +++ b/doc/source/whatsnew/v2.1.0.rst @@ -83,6 +83,7 @@ Other enhancements - :meth:`arrays.DatetimeArray.map`, :meth:`arrays.TimedeltaArray.map` and :meth:`arrays.PeriodArray.map` can now take a ``na_action`` argument (:issue:`51644`) - :meth:`arrays.SparseArray.map` now supports ``na_action`` (:issue:`52096`). - Add :meth:`diff()` and :meth:`round()` for :class:`Index` (:issue:`19708`) +- Add ``"truncate"`` option to ``if_exists`` argument in :meth:`DataFrame.to_sql` which truncates the existing table before inserting new data (:issue:`37210`). - Add dtype of categories to ``repr`` information of :class:`CategoricalDtype` (:issue:`52179`) - Added to the escape mode "latex-math" preserving without escaping all characters between "\(" and "\)" in formatter (:issue:`51903`) - Adding ``engine_kwargs`` parameter to :meth:`DataFrame.read_excel` (:issue:`52214`) diff --git a/pandas/io/sql.py b/pandas/io/sql.py index ebb994f92d8ad..8087453810d7c 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -683,7 +683,7 @@ def to_sql( name: str, con, schema: str | None = None, - if_exists: Literal["fail", "replace", "append"] = "fail", + if_exists: Literal["fail", "replace", "append", "truncate"] = "fail", index: bool = True, index_label: IndexLabel = None, chunksize: int | None = None, @@ -708,10 +708,11 @@ def to_sql( schema : str, optional Name of SQL schema in database to write to (if database flavor supports this). If None, use default schema (default). - if_exists : {'fail', 'replace', 'append'}, default 'fail' + if_exists : {'fail', 'replace', 'append', 'truncate'}, default 'fail' - fail: If table exists, do nothing. - replace: If table exists, drop it, recreate it, and insert data. - append: If table exists, insert data. Create if does not exist. + - truncate: If table exists, truncate the table, then insert data. index : bool, default True Write DataFrame index as a column. index_label : str or sequence, optional @@ -761,7 +762,7 @@ def to_sql( `sqlite3 `__ or `SQLAlchemy `__ """ # noqa: E501 - if if_exists not in ("fail", "replace", "append"): + if if_exists not in ("fail", "replace", "append", "truncate"): raise ValueError(f"'{if_exists}' is not valid for if_exists") if isinstance(frame, Series): @@ -864,7 +865,7 @@ def __init__( pandas_sql_engine, frame=None, index: bool | str | list[str] | None = True, - if_exists: Literal["fail", "replace", "append"] = "fail", + if_exists: Literal["fail", "replace", "append", "truncate"] = "fail", prefix: str = "pandas", index_label=None, schema=None, @@ -915,6 +916,8 @@ def create(self) -> None: if self.if_exists == "replace": self.pd_sql.drop_table(self.name, self.schema) self._execute_create() + elif self.if_exists == "truncate": + self.pd_sql.trunc_table(self.name, self.schema) elif self.if_exists == "append": pass else: @@ -1404,7 +1407,7 @@ def to_sql( self, frame, name: str, - if_exists: Literal["fail", "replace", "append"] = "fail", + if_exists: Literal["fail", "replace", "append", "truncate"] = "fail", index: bool = True, index_label=None, schema=None, @@ -1788,7 +1791,7 @@ def prep_table( self, frame, name: str, - if_exists: Literal["fail", "replace", "append"] = "fail", + if_exists: Literal["fail", "replace", "append", "truncate"] = "fail", index: bool | str | list[str] | None = True, index_label=None, schema=None, @@ -1865,7 +1868,7 @@ def to_sql( self, frame, name: str, - if_exists: Literal["fail", "replace", "append"] = "fail", + if_exists: Literal["fail", "replace", "append", "truncate"] = "fail", index: bool = True, index_label=None, schema: str | None = None, @@ -1883,10 +1886,11 @@ def to_sql( frame : DataFrame name : string Name of SQL table. - if_exists : {'fail', 'replace', 'append'}, default 'fail' + if_exists : {'fail', 'replace', 'append', 'truncate'}, default 'fail' - fail: If table exists, do nothing. - replace: If table exists, drop it, recreate it, and insert data. - append: If table exists, insert data. Create if does not exist. + - truncate: If table exists, truncate the table, then insert data. index : boolean, default True Write DataFrame index as a column. index_label : string or sequence, default None @@ -1981,6 +1985,17 @@ def drop_table(self, table_name: str, schema: str | None = None) -> None: self.get_table(table_name, schema).drop(bind=self.con) self.meta.clear() + def trunc_table(self, table_name: str, schema: str | None = None) -> None: + schema = schema or self.meta.schema + if self.con.engine.name == "sqlite": + raise NotImplementedError("TRUNCATE not supported on sqlite database.") + if self.has_table(table_name, schema): + self.meta.reflect(bind=self.con, only=[table_name], schema=schema) + with self.run_transaction(): + table_to_truncate = self.get_table(table_name, schema) + self.execute(f"TRUNCATE TABLE {table_to_truncate}") + self.meta.clear() + def _create_sql_schema( self, frame: DataFrame, @@ -2409,6 +2424,9 @@ def drop_table(self, name: str, schema: str | None = None) -> None: drop_sql = f"DROP TABLE {_get_valid_sqlite_name(name)}" self.execute(drop_sql) + def trunc_table(self, name: str, schema: str | None = None) -> None: + raise NotImplementedError("TRUNCATE not supported on sqlite database.") + def _create_sql_schema( self, frame, diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 77e7e6f8d6c41..34652cfdfeeab 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -1104,6 +1104,14 @@ def test_to_sql_replace(self, test_frame1): assert num_rows == num_entries + def test_sqlite_truncate_raises(self, test_frame1): + msg = "TRUNCATE not supported on sqlite database." + with pytest.raises(NotImplementedError, match=msg): + # create table + sql.to_sql(test_frame1, "test_frame3", self.conn, if_exists="fail") + # Add to table again + sql.to_sql(test_frame1, "test_frame3", self.conn, if_exists="truncate") + def test_to_sql_append(self, test_frame1): assert sql.to_sql(test_frame1, "test_frame4", self.conn, if_exists="fail") == 4 @@ -2224,6 +2232,25 @@ def _get_index_columns(self, tbl_name): def test_to_sql_save_index(self): self._to_sql_save_index() + def test_to_sql_truncate(self, test_frame1): + if self.flavor == "sqlite": + msg = "TRUNCATE not supported on sqlite database." + with pytest.raises(NotImplementedError, match=msg): + # create table + sql.to_sql(test_frame1, "test_frame3", self.conn, if_exists="fail") + # Add to table again + sql.to_sql(test_frame1, "test_frame3", self.conn, if_exists="truncate") + else: + sql.to_sql(test_frame1, "test_frame3", self.conn, if_exists="fail") + # Add to table again + sql.to_sql(test_frame1, "test_frame3", self.conn, if_exists="truncate") + assert sql.has_table("test_frame3", self.conn) + + num_entries = len(test_frame1) + num_rows = count_rows(self.conn, "test_frame3") + + assert num_rows == num_entries + def test_transactions(self): self._transaction_test()