From f690442431946ce112b22ff0220679d5d9406933 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Wed, 22 Jun 2022 15:06:14 -0700 Subject: [PATCH 1/4] BUG: to_sql with method=callable not returning int raising TypeError --- doc/source/whatsnew/v1.4.3.rst | 2 +- pandas/core/generic.py | 2 +- pandas/io/sql.py | 6 ++++-- pandas/tests/io/test_sql.py | 20 ++++++++++++++++++-- 4 files changed, 24 insertions(+), 6 deletions(-) diff --git a/doc/source/whatsnew/v1.4.3.rst b/doc/source/whatsnew/v1.4.3.rst index de15a318ed547..7fdda12028e9d 100644 --- a/doc/source/whatsnew/v1.4.3.rst +++ b/doc/source/whatsnew/v1.4.3.rst @@ -41,7 +41,7 @@ Bug fixes ~~~~~~~~~ - Bug in :meth:`pd.eval`, :meth:`DataFrame.eval` and :meth:`DataFrame.query` where passing empty ``local_dict`` or ``global_dict`` was treated as passing ``None`` (:issue:`47084`) - Most I/O methods do no longer suppress ``OSError`` and ``ValueError`` when closing file handles (:issue:`47136`) -- +- Bug in :meth:`DataFrame.to_sql` when ``method`` was a ``callable`` that did not return an ``int`` and would raise a ``TypeError`` (:issue:`46891`) .. --------------------------------------------------------------------------- diff --git a/pandas/core/generic.py b/pandas/core/generic.py index 589ea6e67d926..f23fb15d05794 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -2752,7 +2752,7 @@ def to_sql( ------- None or int Number of rows affected by to_sql. None is returned if the callable - passed into ``method`` does not return the number of rows. + passed into ``method`` does not return an integer number of rows. The number of returned rows affected is the sum of the ``rowcount`` attribute of ``sqlite3.Cursor`` or SQLAlchemy connectable which may not diff --git a/pandas/io/sql.py b/pandas/io/sql.py index 701642ad2cfe2..eddc784269c3b 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -36,6 +36,7 @@ from pandas.core.dtypes.common import ( is_datetime64tz_dtype, is_dict_like, + is_integer, is_list_like, ) from pandas.core.dtypes.dtypes import DatetimeTZDtype @@ -662,7 +663,7 @@ def to_sql( ------- None or int Number of rows affected by to_sql. None is returned if the callable - passed into ``method`` does not return the number of rows. + passed into ``method`` does not return an integer number of rows. .. versionadded:: 1.4.0 @@ -937,7 +938,8 @@ def insert( chunk_iter = zip(*(arr[start_i:end_i] for arr in data_list)) num_inserted = exec_insert(conn, keys, chunk_iter) - if num_inserted is None: + # GH 46891 + if not is_integer(num_inserted): total_inserted = None else: total_inserted += num_inserted diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index e28901fa1a1ed..92592314efb33 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -620,7 +620,8 @@ def test_read_procedure(conn, request): @pytest.mark.db @pytest.mark.parametrize("conn", postgresql_connectable) -def test_copy_from_callable_insertion_method(conn, request): +@pytest.mark.parametrize("expected_count", [2, "Success!"]) +def test_copy_from_callable_insertion_method(conn, expected_count, request): # GH 8953 # Example in io.rst found under _io.sql.method # not available in sqlite, mysql @@ -641,14 +642,29 @@ def psql_insert_copy(table, conn, keys, data_iter): sql_query = f"COPY {table_name} ({columns}) FROM STDIN WITH CSV" cur.copy_expert(sql=sql_query, file=s_buf) + return expected_count conn = request.getfixturevalue(conn) expected = DataFrame({"col1": [1, 2], "col2": [0.1, 0.2], "col3": ["a", "n"]}) - expected.to_sql("test_frame", conn, index=False, method=psql_insert_copy) + result_count = expected.to_sql( + "test_frame", conn, index=False, method=psql_insert_copy + ) + # GH 46891 + if not isinstance(expected_count, int): + assert result_count is None + else: + assert result_count == expected_count result = sql.read_sql_table("test_frame", conn) tm.assert_frame_equal(result, expected) +@pytest.mark.db +@pytest.mark.parametrize("conn", postgresql_connectable) +def test_to_sql_callable_method_count_only_if_return_int(conn, request): + # GH 46891 + pass + + class MixInBase: def teardown_method(self): # if setup fails, there may not be a connection to close. From 45ed5ae40e88a9f1e7184489ba563199486ec855 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Wed, 22 Jun 2022 15:07:53 -0700 Subject: [PATCH 2/4] Remove old scaffold test --- pandas/tests/io/test_sql.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 92592314efb33..dd3464ccf9f64 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -658,13 +658,6 @@ def psql_insert_copy(table, conn, keys, data_iter): tm.assert_frame_equal(result, expected) -@pytest.mark.db -@pytest.mark.parametrize("conn", postgresql_connectable) -def test_to_sql_callable_method_count_only_if_return_int(conn, request): - # GH 46891 - pass - - class MixInBase: def teardown_method(self): # if setup fails, there may not be a connection to close. From da29ba403916223d8942014e7f492d9c8d45c22b Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Wed, 22 Jun 2022 20:48:59 -0700 Subject: [PATCH 3/4] Write in a way to make mypy happy --- pandas/io/sql.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pandas/io/sql.py b/pandas/io/sql.py index eddc784269c3b..cd5e82bcff380 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -928,7 +928,7 @@ def insert( raise ValueError("chunksize argument should be non-zero") chunks = (nrows // chunksize) + 1 - total_inserted = 0 + total_inserted = None with self.pd_sql.run_transaction() as conn: for i in range(chunks): start_i = i * chunksize @@ -939,10 +939,11 @@ def insert( chunk_iter = zip(*(arr[start_i:end_i] for arr in data_list)) num_inserted = exec_insert(conn, keys, chunk_iter) # GH 46891 - if not is_integer(num_inserted): - total_inserted = None - else: - total_inserted += num_inserted + if is_integer(num_inserted): + if total_inserted is None: + total_inserted = num_inserted + else: + total_inserted += num_inserted return total_inserted def _query_iterator( From a3cf8b09cade5cbab6fb304e1f3d22a574c67ba8 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Thu, 23 Jun 2022 16:07:59 -0700 Subject: [PATCH 4/4] Add whatsnew to 1.4.4 --- doc/source/whatsnew/v1.4.4.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v1.4.4.rst b/doc/source/whatsnew/v1.4.4.rst index b462f0c6a8ffe..d54dbf6885bd7 100644 --- a/doc/source/whatsnew/v1.4.4.rst +++ b/doc/source/whatsnew/v1.4.4.rst @@ -14,7 +14,7 @@ including other versions of pandas. Fixed regressions ~~~~~~~~~~~~~~~~~ -- +- Bug in :meth:`DataFrame.to_sql` when ``method`` was a ``callable`` that did not return an ``int`` and would raise a ``TypeError`` (:issue:`46891`) - .. ---------------------------------------------------------------------------