From 6aae5c6639243ee352ab73186a617539d924f750 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Thu, 26 Oct 2023 11:57:48 +0200 Subject: [PATCH] REGR: fix roundtripping datetimes with sqlite type detection (#55690) Co-authored-by: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> (cherry picked from commit aeb3644567a10296cc09aed03bf7e0a3780e67ee) --- doc/source/whatsnew/v2.1.2.rst | 1 + pandas/io/sql.py | 6 ++---- pandas/tests/io/test_sql.py | 18 ++++++++++++++++++ 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/doc/source/whatsnew/v2.1.2.rst b/doc/source/whatsnew/v2.1.2.rst index 16f8e8d264268..89a0e4bc5c403 100644 --- a/doc/source/whatsnew/v2.1.2.rst +++ b/doc/source/whatsnew/v2.1.2.rst @@ -20,6 +20,7 @@ Fixed regressions - Fixed regression in :meth:`DataFrameGroupBy.agg` and :meth:`SeriesGroupBy.agg` where if the option ``compute.use_numba`` was set to True, groupby methods not supported by the numba engine would raise a ``TypeError`` (:issue:`55520`) - Fixed performance regression with wide DataFrames, typically involving methods where all columns were accessed individually (:issue:`55256`, :issue:`55245`) - Fixed regression in :func:`merge_asof` raising ``TypeError`` for ``by`` with datetime and timedelta dtypes (:issue:`55453`) +- Fixed regression in :meth:`DataFrame.to_sql` not roundtripping datetime columns correctly for sqlite when using ``detect_types`` (:issue:`55554`) .. --------------------------------------------------------------------------- .. _whatsnew_212.bug_fixes: diff --git a/pandas/io/sql.py b/pandas/io/sql.py index 2b139f8ca527c..c1d68d71ac91c 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -2090,7 +2090,7 @@ def _adapt_time(t) -> str: # Python 3.12+ doesn't auto-register adapters for us anymore adapt_date_iso = lambda val: val.isoformat() - adapt_datetime_iso = lambda val: val.isoformat() + adapt_datetime_iso = lambda val: val.isoformat(" ") sqlite3.register_adapter(time, _adapt_time) @@ -2098,11 +2098,9 @@ def _adapt_time(t) -> str: sqlite3.register_adapter(datetime, adapt_datetime_iso) convert_date = lambda val: date.fromisoformat(val.decode()) - convert_datetime = lambda val: datetime.fromisoformat(val.decode()) - convert_timestamp = lambda val: datetime.fromtimestamp(int(val)) + convert_timestamp = lambda val: datetime.fromisoformat(val.decode()) sqlite3.register_converter("date", convert_date) - sqlite3.register_converter("datetime", convert_datetime) sqlite3.register_converter("timestamp", convert_timestamp) def sql_schema(self) -> str: diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index bfa93a4ff910e..5fd6a52031c52 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -2970,6 +2970,24 @@ def test_roundtripping_datetimes(self): assert result == "2020-12-31 12:00:00.000000" +@pytest.fixture +def sqlite_builtin_detect_types(): + with contextlib.closing( + sqlite3.connect(":memory:", detect_types=sqlite3.PARSE_DECLTYPES) + ) as closing_conn: + with closing_conn as conn: + yield conn + + +def test_roundtripping_datetimes_detect_types(sqlite_builtin_detect_types): + # https://github.com/pandas-dev/pandas/issues/55554 + conn = sqlite_builtin_detect_types + df = DataFrame({"t": [datetime(2020, 12, 31, 12)]}, dtype="datetime64[ns]") + df.to_sql("test", conn, if_exists="replace", index=False) + result = pd.read_sql("select * from test", conn).iloc[0, 0] + assert result == Timestamp("2020-12-31 12:00:00.000000") + + @pytest.mark.db class TestMySQLAlchemy(_TestSQLAlchemy): """