From dbd141ab2968218937a160c79aa744048462fb33 Mon Sep 17 00:00:00 2001 From: arw2019 Date: Sun, 31 May 2020 03:13:39 +0000 Subject: [PATCH 01/33] TST: pd.to_sql for dataframes with -np.inf (#34431) --- pandas/tests/io/test_sql.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index bd53785e89bfe..f537c86958b7d 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -564,6 +564,13 @@ class DummyException(Exception): res2 = self.pandasSQL.read_query("SELECT * FROM test_trans") assert len(res2) == 1 + def _to_sql_with_negative_npinf(self): + engine = sqlalchemy.create_engine("sqlite://", echo=False) + + pd.DataFrame({"foo": [np.inf]}).to_sql("foobar1", engine) + pd.DataFrame({"foo": [-np.inf]}).to_sql("foobar2", engine) + pd.DataFrame({"foo": [-np.inf], "infe0": ["bar"]}).to_sql("foobar3", engine) + # ----------------------------------------------------------------------------- # -- Testing the public API From 70f3e12bfde9a78461c8c8b9c7522df518d1129d Mon Sep 17 00:00:00 2001 From: arw2019 Date: Sun, 31 May 2020 03:27:55 +0000 Subject: [PATCH 02/33] DOC: updated what's new (#34431) --- doc/source/whatsnew/v1.1.0.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/whatsnew/v1.1.0.rst b/doc/source/whatsnew/v1.1.0.rst index 88bf0e005a221..04a097eec02d6 100644 --- a/doc/source/whatsnew/v1.1.0.rst +++ b/doc/source/whatsnew/v1.1.0.rst @@ -901,6 +901,7 @@ I/O - Bug in :meth:`~DataFrame.read_feather` was raising an `ArrowIOError` when reading an s3 or http file path (:issue:`29055`) - Bug in :meth:`~DataFrame.to_excel` could not handle the column name `render` and was raising an ``KeyError`` (:issue:`34331`) - Bug in :meth:`~SQLDatabase.execute` was raising a ``ProgrammingError`` for some DB-API drivers when the SQL statement contained the `%` character and no parameters were present (:issue:`34211`) +- Bug in :meth: in earlier versions `pd.to_sql` was raising an error when reading DataFrames with `-np.inf` entries; added test (:issue: `34431`) Plotting ^^^^^^^^ From b395497b6c1f522a066a49d356f71a884a74cfb8 Mon Sep 17 00:00:00 2001 From: arw2019 Date: Mon, 1 Jun 2020 02:08:42 +0000 Subject: [PATCH 03/33] DOC: improved entry (#34431) --- doc/source/whatsnew/v1.1.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v1.1.0.rst b/doc/source/whatsnew/v1.1.0.rst index 04a097eec02d6..72e6795da1302 100644 --- a/doc/source/whatsnew/v1.1.0.rst +++ b/doc/source/whatsnew/v1.1.0.rst @@ -901,7 +901,7 @@ I/O - Bug in :meth:`~DataFrame.read_feather` was raising an `ArrowIOError` when reading an s3 or http file path (:issue:`29055`) - Bug in :meth:`~DataFrame.to_excel` could not handle the column name `render` and was raising an ``KeyError`` (:issue:`34331`) - Bug in :meth:`~SQLDatabase.execute` was raising a ``ProgrammingError`` for some DB-API drivers when the SQL statement contained the `%` character and no parameters were present (:issue:`34211`) -- Bug in :meth: in earlier versions `pd.to_sql` was raising an error when reading DataFrames with `-np.inf` entries; added test (:issue: `34431`) +- Bug in :meth:`DataFrame.to_sql` when reading DataFrames with `-np.inf` entries (:issue: `34431`) Plotting ^^^^^^^^ From f76e5d3bba63f9d8d5a0f92bb18d393f022cb73d Mon Sep 17 00:00:00 2001 From: arw2019 Date: Mon, 1 Jun 2020 03:10:10 +0000 Subject: [PATCH 04/33] TST: moved to _TestSQLAlchemy + added round trips --- pandas/tests/io/test_sql.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index f537c86958b7d..11e79fedb685b 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -564,13 +564,6 @@ class DummyException(Exception): res2 = self.pandasSQL.read_query("SELECT * FROM test_trans") assert len(res2) == 1 - def _to_sql_with_negative_npinf(self): - engine = sqlalchemy.create_engine("sqlite://", echo=False) - - pd.DataFrame({"foo": [np.inf]}).to_sql("foobar1", engine) - pd.DataFrame({"foo": [-np.inf]}).to_sql("foobar2", engine) - pd.DataFrame({"foo": [-np.inf], "infe0": ["bar"]}).to_sql("foobar3", engine) - # ----------------------------------------------------------------------------- # -- Testing the public API @@ -1820,6 +1813,22 @@ def main(connectable): DataFrame({"test_foo_data": [0, 1, 2]}).to_sql("test_foo_data", self.conn) main(self.conn) + def _to_sql_with_negative_npinf(self): + df1 = pd.DataFrame({"foo": [np.inf]}) + df1.to_sql("foobar1", self.conn, index=False) + res1 = sql.read_sql_table("foobar1", self.conn) + tm.assert_equal(df1, res1) + + df2 = pd.DataFrame({"foo": [-np.inf]}) + df2.to_sql("foobar2", self.conn, index=False) + res2 = sql.read_sql_table("foobar2", self.conn) + tm.assert_equal(df2, res2) + + df3 = pd.DataFrame({"foo": [-np.inf], "infe0": ["bar"]}) + df3.to_sql("foobar3", self.conn, index=False) + res3 = sql.read_sql_table("foobar3", self.conn) + tm.assert_equal(df3, res3) + def test_temporary_table(self): test_data = "Hello, World!" expected = DataFrame({"spam": [test_data]}) From b205b1f6c866d83719d2105971aa4837c796d1e7 Mon Sep 17 00:00:00 2001 From: arw2019 Date: Mon, 1 Jun 2020 03:32:32 +0000 Subject: [PATCH 05/33] TST: rename + add comment with GH issue # --- pandas/tests/io/test_sql.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 11e79fedb685b..77891b5abdf95 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -1813,7 +1813,9 @@ def main(connectable): DataFrame({"test_foo_data": [0, 1, 2]}).to_sql("test_foo_data", self.conn) main(self.conn) - def _to_sql_with_negative_npinf(self): + def test_to_sql_with_negative_npinf(self): + # This tests the example raised in GH issue 34431. + df1 = pd.DataFrame({"foo": [np.inf]}) df1.to_sql("foobar1", self.conn, index=False) res1 = sql.read_sql_table("foobar1", self.conn) From 3c91ce1e1fb898941b75e8287d127549593afbb2 Mon Sep 17 00:00:00 2001 From: arw2019 Date: Mon, 1 Jun 2020 04:28:51 +0000 Subject: [PATCH 06/33] TST: rewrote using pytest.mark.parametrize for arg to DataFrame --- pandas/tests/io/test_sql.py | 28 +++++++++++----------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 77891b5abdf95..1ed5c766f93ee 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -1813,23 +1813,17 @@ def main(connectable): DataFrame({"test_foo_data": [0, 1, 2]}).to_sql("test_foo_data", self.conn) main(self.conn) - def test_to_sql_with_negative_npinf(self): - # This tests the example raised in GH issue 34431. - - df1 = pd.DataFrame({"foo": [np.inf]}) - df1.to_sql("foobar1", self.conn, index=False) - res1 = sql.read_sql_table("foobar1", self.conn) - tm.assert_equal(df1, res1) - - df2 = pd.DataFrame({"foo": [-np.inf]}) - df2.to_sql("foobar2", self.conn, index=False) - res2 = sql.read_sql_table("foobar2", self.conn) - tm.assert_equal(df2, res2) - - df3 = pd.DataFrame({"foo": [-np.inf], "infe0": ["bar"]}) - df3.to_sql("foobar3", self.conn, index=False) - res3 = sql.read_sql_table("foobar3", self.conn) - tm.assert_equal(df3, res3) + @pytest.mark.parametrize( + "_input", + [{"foo": [np.inf]}, {"foo": [-np.inf]}, {"foo": [-np.inf], "infe0": ["bar"]}], + ) + def test_to_sql_with_negative_npinf(self, _input): + # GH 34431 + + df = pd.DataFrame(_input) + df.to_sql("foobar", self.conn, index=False) + res = sql.read_sql_table("foobar", self.conn) + tm.assert_equal(df, res) def test_temporary_table(self): test_data = "Hello, World!" From d3937ef516fa63223f1908a44fc69c0c1cfee0af Mon Sep 17 00:00:00 2001 From: arw2019 Date: Mon, 1 Jun 2020 18:43:46 +0000 Subject: [PATCH 07/33] TST: removed underscore from _input --- pandas/tests/io/test_sql.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 1ed5c766f93ee..e0b3f7c930f06 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -1814,13 +1814,13 @@ def main(connectable): main(self.conn) @pytest.mark.parametrize( - "_input", + "input", [{"foo": [np.inf]}, {"foo": [-np.inf]}, {"foo": [-np.inf], "infe0": ["bar"]}], ) - def test_to_sql_with_negative_npinf(self, _input): + def test_to_sql_with_negative_npinf(self, input): # GH 34431 - df = pd.DataFrame(_input) + df = pd.DataFrame(input) df.to_sql("foobar", self.conn, index=False) res = sql.read_sql_table("foobar", self.conn) tm.assert_equal(df, res) From a836965d06a1dfd399c694e64298c715f6a041fc Mon Sep 17 00:00:00 2001 From: arw2019 Date: Mon, 1 Jun 2020 18:46:07 +0000 Subject: [PATCH 08/33] DOC: added double backtick to np.inf & removed extraneous space --- doc/source/whatsnew/v1.1.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v1.1.0.rst b/doc/source/whatsnew/v1.1.0.rst index 72e6795da1302..5325821ff9ae8 100644 --- a/doc/source/whatsnew/v1.1.0.rst +++ b/doc/source/whatsnew/v1.1.0.rst @@ -901,7 +901,7 @@ I/O - Bug in :meth:`~DataFrame.read_feather` was raising an `ArrowIOError` when reading an s3 or http file path (:issue:`29055`) - Bug in :meth:`~DataFrame.to_excel` could not handle the column name `render` and was raising an ``KeyError`` (:issue:`34331`) - Bug in :meth:`~SQLDatabase.execute` was raising a ``ProgrammingError`` for some DB-API drivers when the SQL statement contained the `%` character and no parameters were present (:issue:`34211`) -- Bug in :meth:`DataFrame.to_sql` when reading DataFrames with `-np.inf` entries (:issue: `34431`) +- Bug in :meth:`DataFrame.to_sql` when reading DataFrames with ``-np.inf`` entries (:issue:`34431`) Plotting ^^^^^^^^ From dfbf1c75c34fd59d101d2218039788339ebd8d4b Mon Sep 17 00:00:00 2001 From: arw2019 Date: Sun, 31 May 2020 03:13:39 +0000 Subject: [PATCH 09/33] TST: pd.to_sql for dataframes with -np.inf (#34431) --- pandas/tests/io/test_sql.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index bd53785e89bfe..f537c86958b7d 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -564,6 +564,13 @@ class DummyException(Exception): res2 = self.pandasSQL.read_query("SELECT * FROM test_trans") assert len(res2) == 1 + def _to_sql_with_negative_npinf(self): + engine = sqlalchemy.create_engine("sqlite://", echo=False) + + pd.DataFrame({"foo": [np.inf]}).to_sql("foobar1", engine) + pd.DataFrame({"foo": [-np.inf]}).to_sql("foobar2", engine) + pd.DataFrame({"foo": [-np.inf], "infe0": ["bar"]}).to_sql("foobar3", engine) + # ----------------------------------------------------------------------------- # -- Testing the public API From 5a5ce91dba4071c1dbb9531c220ee5a6ae63de03 Mon Sep 17 00:00:00 2001 From: arw2019 Date: Sun, 31 May 2020 03:27:55 +0000 Subject: [PATCH 10/33] DOC: updated what's new (#34431) --- doc/source/whatsnew/v1.1.0.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/whatsnew/v1.1.0.rst b/doc/source/whatsnew/v1.1.0.rst index 7834e1a5c4898..55c00452d6b60 100644 --- a/doc/source/whatsnew/v1.1.0.rst +++ b/doc/source/whatsnew/v1.1.0.rst @@ -949,6 +949,7 @@ I/O - Bug in :meth:`~DataFrame.read_feather` was raising an `ArrowIOError` when reading an s3 or http file path (:issue:`29055`) - Bug in :meth:`~DataFrame.to_excel` could not handle the column name `render` and was raising an ``KeyError`` (:issue:`34331`) - Bug in :meth:`~SQLDatabase.execute` was raising a ``ProgrammingError`` for some DB-API drivers when the SQL statement contained the `%` character and no parameters were present (:issue:`34211`) +- Bug in :meth: in earlier versions `pd.to_sql` was raising an error when reading DataFrames with `-np.inf` entries; added test (:issue: `34431`) Plotting ^^^^^^^^ From 45f2f1e6740779ad7322fc33ec9a1e6274d99095 Mon Sep 17 00:00:00 2001 From: arw2019 Date: Mon, 1 Jun 2020 02:08:42 +0000 Subject: [PATCH 11/33] DOC: improved entry (#34431) --- doc/source/whatsnew/v1.1.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v1.1.0.rst b/doc/source/whatsnew/v1.1.0.rst index 55c00452d6b60..087e933eec5f8 100644 --- a/doc/source/whatsnew/v1.1.0.rst +++ b/doc/source/whatsnew/v1.1.0.rst @@ -949,7 +949,7 @@ I/O - Bug in :meth:`~DataFrame.read_feather` was raising an `ArrowIOError` when reading an s3 or http file path (:issue:`29055`) - Bug in :meth:`~DataFrame.to_excel` could not handle the column name `render` and was raising an ``KeyError`` (:issue:`34331`) - Bug in :meth:`~SQLDatabase.execute` was raising a ``ProgrammingError`` for some DB-API drivers when the SQL statement contained the `%` character and no parameters were present (:issue:`34211`) -- Bug in :meth: in earlier versions `pd.to_sql` was raising an error when reading DataFrames with `-np.inf` entries; added test (:issue: `34431`) +- Bug in :meth:`DataFrame.to_sql` when reading DataFrames with `-np.inf` entries (:issue: `34431`) Plotting ^^^^^^^^ From e5e500b5bdd55a3c65b4c6182a9ed4e8ce28a143 Mon Sep 17 00:00:00 2001 From: arw2019 Date: Mon, 1 Jun 2020 03:10:10 +0000 Subject: [PATCH 12/33] TST: moved to _TestSQLAlchemy + added round trips --- pandas/tests/io/test_sql.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index f537c86958b7d..11e79fedb685b 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -564,13 +564,6 @@ class DummyException(Exception): res2 = self.pandasSQL.read_query("SELECT * FROM test_trans") assert len(res2) == 1 - def _to_sql_with_negative_npinf(self): - engine = sqlalchemy.create_engine("sqlite://", echo=False) - - pd.DataFrame({"foo": [np.inf]}).to_sql("foobar1", engine) - pd.DataFrame({"foo": [-np.inf]}).to_sql("foobar2", engine) - pd.DataFrame({"foo": [-np.inf], "infe0": ["bar"]}).to_sql("foobar3", engine) - # ----------------------------------------------------------------------------- # -- Testing the public API @@ -1820,6 +1813,22 @@ def main(connectable): DataFrame({"test_foo_data": [0, 1, 2]}).to_sql("test_foo_data", self.conn) main(self.conn) + def _to_sql_with_negative_npinf(self): + df1 = pd.DataFrame({"foo": [np.inf]}) + df1.to_sql("foobar1", self.conn, index=False) + res1 = sql.read_sql_table("foobar1", self.conn) + tm.assert_equal(df1, res1) + + df2 = pd.DataFrame({"foo": [-np.inf]}) + df2.to_sql("foobar2", self.conn, index=False) + res2 = sql.read_sql_table("foobar2", self.conn) + tm.assert_equal(df2, res2) + + df3 = pd.DataFrame({"foo": [-np.inf], "infe0": ["bar"]}) + df3.to_sql("foobar3", self.conn, index=False) + res3 = sql.read_sql_table("foobar3", self.conn) + tm.assert_equal(df3, res3) + def test_temporary_table(self): test_data = "Hello, World!" expected = DataFrame({"spam": [test_data]}) From e78cf404c38518dc9c2fe10900ac41a69d818e4a Mon Sep 17 00:00:00 2001 From: arw2019 Date: Mon, 1 Jun 2020 03:32:32 +0000 Subject: [PATCH 13/33] TST: rename + add comment with GH issue # --- pandas/tests/io/test_sql.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 11e79fedb685b..77891b5abdf95 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -1813,7 +1813,9 @@ def main(connectable): DataFrame({"test_foo_data": [0, 1, 2]}).to_sql("test_foo_data", self.conn) main(self.conn) - def _to_sql_with_negative_npinf(self): + def test_to_sql_with_negative_npinf(self): + # This tests the example raised in GH issue 34431. + df1 = pd.DataFrame({"foo": [np.inf]}) df1.to_sql("foobar1", self.conn, index=False) res1 = sql.read_sql_table("foobar1", self.conn) From e0067319d7e0beff150211b57278711f2b0e6e84 Mon Sep 17 00:00:00 2001 From: arw2019 Date: Mon, 1 Jun 2020 04:28:51 +0000 Subject: [PATCH 14/33] TST: rewrote using pytest.mark.parametrize for arg to DataFrame --- pandas/tests/io/test_sql.py | 28 +++++++++++----------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 77891b5abdf95..1ed5c766f93ee 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -1813,23 +1813,17 @@ def main(connectable): DataFrame({"test_foo_data": [0, 1, 2]}).to_sql("test_foo_data", self.conn) main(self.conn) - def test_to_sql_with_negative_npinf(self): - # This tests the example raised in GH issue 34431. - - df1 = pd.DataFrame({"foo": [np.inf]}) - df1.to_sql("foobar1", self.conn, index=False) - res1 = sql.read_sql_table("foobar1", self.conn) - tm.assert_equal(df1, res1) - - df2 = pd.DataFrame({"foo": [-np.inf]}) - df2.to_sql("foobar2", self.conn, index=False) - res2 = sql.read_sql_table("foobar2", self.conn) - tm.assert_equal(df2, res2) - - df3 = pd.DataFrame({"foo": [-np.inf], "infe0": ["bar"]}) - df3.to_sql("foobar3", self.conn, index=False) - res3 = sql.read_sql_table("foobar3", self.conn) - tm.assert_equal(df3, res3) + @pytest.mark.parametrize( + "_input", + [{"foo": [np.inf]}, {"foo": [-np.inf]}, {"foo": [-np.inf], "infe0": ["bar"]}], + ) + def test_to_sql_with_negative_npinf(self, _input): + # GH 34431 + + df = pd.DataFrame(_input) + df.to_sql("foobar", self.conn, index=False) + res = sql.read_sql_table("foobar", self.conn) + tm.assert_equal(df, res) def test_temporary_table(self): test_data = "Hello, World!" From 1a312df3bb2639e6f10bd4ffd7f7ccb404f2de3d Mon Sep 17 00:00:00 2001 From: arw2019 Date: Mon, 1 Jun 2020 18:43:46 +0000 Subject: [PATCH 15/33] TST: removed underscore from _input --- pandas/tests/io/test_sql.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 1ed5c766f93ee..e0b3f7c930f06 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -1814,13 +1814,13 @@ def main(connectable): main(self.conn) @pytest.mark.parametrize( - "_input", + "input", [{"foo": [np.inf]}, {"foo": [-np.inf]}, {"foo": [-np.inf], "infe0": ["bar"]}], ) - def test_to_sql_with_negative_npinf(self, _input): + def test_to_sql_with_negative_npinf(self, input): # GH 34431 - df = pd.DataFrame(_input) + df = pd.DataFrame(input) df.to_sql("foobar", self.conn, index=False) res = sql.read_sql_table("foobar", self.conn) tm.assert_equal(df, res) From 8e55729a88edcf60e179a4e6879db5fe244a4616 Mon Sep 17 00:00:00 2001 From: arw2019 Date: Mon, 1 Jun 2020 18:46:07 +0000 Subject: [PATCH 16/33] DOC: added double backtick to np.inf & removed extraneous space --- doc/source/whatsnew/v1.1.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v1.1.0.rst b/doc/source/whatsnew/v1.1.0.rst index 087e933eec5f8..29f331f79c94e 100644 --- a/doc/source/whatsnew/v1.1.0.rst +++ b/doc/source/whatsnew/v1.1.0.rst @@ -949,7 +949,7 @@ I/O - Bug in :meth:`~DataFrame.read_feather` was raising an `ArrowIOError` when reading an s3 or http file path (:issue:`29055`) - Bug in :meth:`~DataFrame.to_excel` could not handle the column name `render` and was raising an ``KeyError`` (:issue:`34331`) - Bug in :meth:`~SQLDatabase.execute` was raising a ``ProgrammingError`` for some DB-API drivers when the SQL statement contained the `%` character and no parameters were present (:issue:`34211`) -- Bug in :meth:`DataFrame.to_sql` when reading DataFrames with `-np.inf` entries (:issue: `34431`) +- Bug in :meth:`DataFrame.to_sql` when reading DataFrames with ``-np.inf`` entries (:issue:`34431`) Plotting ^^^^^^^^ From 51b2c6a2caf310619457b1b52a960cfe08c7c487 Mon Sep 17 00:00:00 2001 From: arw2019 Date: Tue, 23 Jun 2020 22:10:26 +0000 Subject: [PATCH 17/33] BUG: add catch for MySQL error with np.inf --- pandas/io/sql.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/pandas/io/sql.py b/pandas/io/sql.py index b137608475b3d..07fd99c22f630 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -11,6 +11,7 @@ import warnings import numpy as np +from sqlalchemy import exc import pandas._libs.lib as lib @@ -1391,7 +1392,17 @@ def to_sql( dtype=dtype, ) table.create() - table.insert(chunksize, method=method) + try: + table.insert(chunksize, method=method) + except exc.SQLAlchemyError as err: + if ( + str(err.orig) + == '(1054, "Unknown column ' + "'inf' in 'field list'" + '")' + ): + raise ValueError("ProgrammingError: inf can not be used with MySQL") + else: + raise err + if not name.isdigit() and not name.islower(): # check for potentially case sensitivity issues (GH7815) # Only check when name is not a number and name is not lower case From cc6d53bb535e68dbbedfe35b7d2f2cf23b2985ab Mon Sep 17 00:00:00 2001 From: arw2019 Date: Tue, 23 Jun 2020 23:04:00 +0000 Subject: [PATCH 18/33] use regex for string match + add runtime import --- pandas/io/sql.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pandas/io/sql.py b/pandas/io/sql.py index 07fd99c22f630..3c8d088dfe3d5 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -11,7 +11,6 @@ import warnings import numpy as np -from sqlalchemy import exc import pandas._libs.lib as lib @@ -1392,16 +1391,17 @@ def to_sql( dtype=dtype, ) table.create() + + from sqlalchemy import exc + try: table.insert(chunksize, method=method) except exc.SQLAlchemyError as err: - if ( - str(err.orig) - == '(1054, "Unknown column ' + "'inf' in 'field list'" + '")' - ): - raise ValueError("ProgrammingError: inf can not be used with MySQL") + msg = '(1054, "Unknown column ' + "'inf' in 'field list'" + '")' + if re.search(msg, str(err.orig)): + raise ValueError("inf can not be used with MySQL") else: - raise err + raise ValueError(err) if not name.isdigit() and not name.islower(): # check for potentially case sensitivity issues (GH7815) From e88222810de1240dd8bdfdefc203450b620fba9a Mon Sep 17 00:00:00 2001 From: arw2019 Date: Tue, 23 Jun 2020 23:06:02 +0000 Subject: [PATCH 19/33] clean up regex --- pandas/io/sql.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/io/sql.py b/pandas/io/sql.py index 3c8d088dfe3d5..75e1fe587239c 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -1398,7 +1398,8 @@ def to_sql( table.insert(chunksize, method=method) except exc.SQLAlchemyError as err: msg = '(1054, "Unknown column ' + "'inf' in 'field list'" + '")' - if re.search(msg, str(err.orig)): + errText = str(err.orig) + if re.search(msg, errText): raise ValueError("inf can not be used with MySQL") else: raise ValueError(err) From 9c828ccb7d0dd5338454c3340f077748c8b3a581 Mon Sep 17 00:00:00 2001 From: arw2019 Date: Tue, 23 Jun 2020 23:09:02 +0000 Subject: [PATCH 20/33] TST: update to catch error for -np.inf with MySQL --- pandas/tests/io/test_sql.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index ef191156c23c0..3b48d95818e85 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -1822,8 +1822,11 @@ def test_to_sql_with_negative_npinf(self, input): df = pd.DataFrame(input) df.to_sql("foobar", self.conn, index=False) - res = sql.read_sql_table("foobar", self.conn) - tm.assert_equal(df, res) + try: + res = sql.read_sql_table("foobar", self.conn) + tm.assert_equal(df, res) + except ValueError("inf can not be used with MySQL"): + pass def test_temporary_table(self): test_data = "Hello, World!" From 7c0a49c06b8880598e66f9d1115e875402f3be13 Mon Sep 17 00:00:00 2001 From: arw2019 Date: Tue, 23 Jun 2020 23:11:26 +0000 Subject: [PATCH 21/33] DOC: resolved conflict in whatsnew --- doc/source/whatsnew/v1.1.0.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/doc/source/whatsnew/v1.1.0.rst b/doc/source/whatsnew/v1.1.0.rst index b585d6a31626a..09e71205adab3 100644 --- a/doc/source/whatsnew/v1.1.0.rst +++ b/doc/source/whatsnew/v1.1.0.rst @@ -994,11 +994,8 @@ I/O - Bug in :meth:`~DataFrame.read_feather` was raising an `ArrowIOError` when reading an s3 or http file path (:issue:`29055`) - Bug in :meth:`~DataFrame.to_excel` could not handle the column name `render` and was raising an ``KeyError`` (:issue:`34331`) - Bug in :meth:`~SQLDatabase.execute` was raising a ``ProgrammingError`` for some DB-API drivers when the SQL statement contained the `%` character and no parameters were present (:issue:`34211`) -<<<<<<< HEAD -======= - Bug in :meth:`~pandas.io.stata.StataReader` which resulted in categorical variables with difference dtypes when reading data using an iterator. (:issue:`31544`) - :meth:`HDFStore.keys` has now an optional `include` parameter that allows the retrieval of all native HDF5 table names (:issue:`29916`) ->>>>>>> 6677c9e4523fc90686724c5edfd81a9b22c290ec - Bug in :meth:`DataFrame.to_sql` when reading DataFrames with ``-np.inf`` entries (:issue:`34431`) Plotting From 030b9fcb9f30bb468441e4739103cb5d6ef6f512 Mon Sep 17 00:00:00 2001 From: arw2019 Date: Tue, 23 Jun 2020 23:30:49 +0000 Subject: [PATCH 22/33] TST: update test_to_sql_with_neg_npinf --- pandas/tests/io/test_sql.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 3b48d95818e85..9b6ed5c018445 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -20,6 +20,7 @@ import csv from datetime import date, datetime, time from io import StringIO +import re import sqlite3 import warnings @@ -1825,8 +1826,12 @@ def test_to_sql_with_negative_npinf(self, input): try: res = sql.read_sql_table("foobar", self.conn) tm.assert_equal(df, res) - except ValueError("inf can not be used with MySQL"): - pass + except ValueError as err: + mes = "inf can not be used with MySQL" + if re.search(mes, err.args[0]): + pass + else: + raise err def test_temporary_table(self): test_data = "Hello, World!" From 9e63a8027424d2493b78946a4ee3628d80d9c8eb Mon Sep 17 00:00:00 2001 From: arw2019 Date: Wed, 24 Jun 2020 02:41:04 +0000 Subject: [PATCH 23/33] fixed error handler syntax in SQLDatabase.to_sql --- pandas/io/sql.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/io/sql.py b/pandas/io/sql.py index 75e1fe587239c..5733f88aba304 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -1400,9 +1400,9 @@ def to_sql( msg = '(1054, "Unknown column ' + "'inf' in 'field list'" + '")' errText = str(err.orig) if re.search(msg, errText): - raise ValueError("inf can not be used with MySQL") + raise ValueError("inf can not be used with MySQL") from err else: - raise ValueError(err) + raise err if not name.isdigit() and not name.islower(): # check for potentially case sensitivity issues (GH7815) From 550c1105054f6687add6bce610da05e68163515d Mon Sep 17 00:00:00 2001 From: arw2019 Date: Wed, 24 Jun 2020 02:49:20 +0000 Subject: [PATCH 24/33] fixed error handler syntax in SQLDatabase.to_sql --- pandas/io/sql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/io/sql.py b/pandas/io/sql.py index 5733f88aba304..87a6f661953ed 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -1397,7 +1397,7 @@ def to_sql( try: table.insert(chunksize, method=method) except exc.SQLAlchemyError as err: - msg = '(1054, "Unknown column ' + "'inf' in 'field list'" + '")' + msg = "(1054, \"Unknown column 'inf' in 'field list'\")" errText = str(err.orig) if re.search(msg, errText): raise ValueError("inf can not be used with MySQL") from err From fe6dbf0f8cb716d40fe140f1018b191a8cb05090 Mon Sep 17 00:00:00 2001 From: arw2019 Date: Wed, 24 Jun 2020 02:59:12 +0000 Subject: [PATCH 25/33] TST: added an xfail test for npinf entries with mysql --- pandas/tests/io/test_sql.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 9b6ed5c018445..c22396136c121 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -1821,6 +1821,7 @@ def main(connectable): def test_to_sql_with_negative_npinf(self, input): # GH 34431 + print(f"type(self.conn)={type(self.conn)}") df = pd.DataFrame(input) df.to_sql("foobar", self.conn, index=False) try: From 57e740382cf28b1665caadf7d27c5cc848e9cec7 Mon Sep 17 00:00:00 2001 From: arw2019 Date: Wed, 24 Jun 2020 03:02:19 +0000 Subject: [PATCH 26/33] fixed imports --- pandas/tests/io/test_sql.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index c22396136c121..f0cf55e550186 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -20,7 +20,6 @@ import csv from datetime import date, datetime, time from io import StringIO -import re import sqlite3 import warnings @@ -1821,18 +1820,15 @@ def main(connectable): def test_to_sql_with_negative_npinf(self, input): # GH 34431 - print(f"type(self.conn)={type(self.conn)}") df = pd.DataFrame(input) - df.to_sql("foobar", self.conn, index=False) - try: + + if self.conn.dialect.name == "mysql": + with pytest.raises(ValueError("inf can not be used with MySQL")): + df.to_sql("foobar", self.conn, index=False) + else: + df.to_sql("foobar", self.conn, index=False) res = sql.read_sql_table("foobar", self.conn) tm.assert_equal(df, res) - except ValueError as err: - mes = "inf can not be used with MySQL" - if re.search(mes, err.args[0]): - pass - else: - raise err def test_temporary_table(self): test_data = "Hello, World!" From 97f79187634486e12056bbaef5a61432f146c96d Mon Sep 17 00:00:00 2001 From: arw2019 Date: Wed, 24 Jun 2020 03:04:10 +0000 Subject: [PATCH 27/33] added reference to GH issue --- pandas/io/sql.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pandas/io/sql.py b/pandas/io/sql.py index 87a6f661953ed..ee9d5dfbe0606 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -1397,6 +1397,7 @@ def to_sql( try: table.insert(chunksize, method=method) except exc.SQLAlchemyError as err: + # GH34431 msg = "(1054, \"Unknown column 'inf' in 'field list'\")" errText = str(err.orig) if re.search(msg, errText): From a7fc4938b5585fa2219ca1690fe189913d945766 Mon Sep 17 00:00:00 2001 From: arw2019 Date: Wed, 24 Jun 2020 04:57:14 +0000 Subject: [PATCH 28/33] fixed test_to_sql_with_npinf error catch --- pandas/tests/io/test_sql.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index f0cf55e550186..6e3b9dafc4797 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -1823,7 +1823,8 @@ def test_to_sql_with_negative_npinf(self, input): df = pd.DataFrame(input) if self.conn.dialect.name == "mysql": - with pytest.raises(ValueError("inf can not be used with MySQL")): + msg = "inf can not be used with MySQL" + with pytest.raises(ValueError, match=msg): df.to_sql("foobar", self.conn, index=False) else: df.to_sql("foobar", self.conn, index=False) From fa4e9291479034333d01bd713def72a09611e4fd Mon Sep 17 00:00:00 2001 From: arw2019 Date: Wed, 24 Jun 2020 05:00:13 +0000 Subject: [PATCH 29/33] fixed spelling error in message (can not -> cannot) --- pandas/io/sql.py | 2 +- pandas/tests/io/test_sql.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/io/sql.py b/pandas/io/sql.py index ee9d5dfbe0606..0efad28293ae8 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -1401,7 +1401,7 @@ def to_sql( msg = "(1054, \"Unknown column 'inf' in 'field list'\")" errText = str(err.orig) if re.search(msg, errText): - raise ValueError("inf can not be used with MySQL") from err + raise ValueError("inf cannot be used with MySQL") from err else: raise err diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 6e3b9dafc4797..cc6177779e380 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -1823,7 +1823,7 @@ def test_to_sql_with_negative_npinf(self, input): df = pd.DataFrame(input) if self.conn.dialect.name == "mysql": - msg = "inf can not be used with MySQL" + msg = "inf cannot be used with MySQL" with pytest.raises(ValueError, match=msg): df.to_sql("foobar", self.conn, index=False) else: From cb81d5cf5110f49dddf2cbf86444b69585b53020 Mon Sep 17 00:00:00 2001 From: arw2019 Date: Wed, 24 Jun 2020 19:10:07 +0000 Subject: [PATCH 30/33] DOC: added info re MySQL ValueError to whatsnew --- doc/source/whatsnew/v1.1.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v1.1.0.rst b/doc/source/whatsnew/v1.1.0.rst index 606b019007076..b3a69c5b7393a 100644 --- a/doc/source/whatsnew/v1.1.0.rst +++ b/doc/source/whatsnew/v1.1.0.rst @@ -1018,7 +1018,7 @@ I/O - Bug in :meth:`~SQLDatabase.execute` was raising a ``ProgrammingError`` for some DB-API drivers when the SQL statement contained the `%` character and no parameters were present (:issue:`34211`) - Bug in :meth:`~pandas.io.stata.StataReader` which resulted in categorical variables with difference dtypes when reading data using an iterator. (:issue:`31544`) - :meth:`HDFStore.keys` has now an optional `include` parameter that allows the retrieval of all native HDF5 table names (:issue:`29916`) -- Bug in :meth:`DataFrame.to_sql` when reading DataFrames with ``-np.inf`` entries (:issue:`34431`) +- Bug in :meth:`DataFrame.to_sql` when reading DataFrames with ``-np.inf`` entries with MySQL now has a more explicit ``ValueError`` (:issue:`34431`) Plotting ^^^^^^^^ From c8d9c9e0dc29f0b30ad65a2e37379bf7655492b7 Mon Sep 17 00:00:00 2001 From: arw2019 Date: Wed, 24 Jun 2020 19:11:31 +0000 Subject: [PATCH 31/33] fixed variable name in to_sql --- pandas/io/sql.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/io/sql.py b/pandas/io/sql.py index 0efad28293ae8..9177696ca13d6 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -1399,8 +1399,8 @@ def to_sql( except exc.SQLAlchemyError as err: # GH34431 msg = "(1054, \"Unknown column 'inf' in 'field list'\")" - errText = str(err.orig) - if re.search(msg, errText): + err_text = str(err.orig) + if re.search(msg, err_text): raise ValueError("inf cannot be used with MySQL") from err else: raise err From 7128d10e34a1e995956c564e903dbcb264c247b0 Mon Sep 17 00:00:00 2001 From: arw2019 Date: Wed, 24 Jun 2020 19:30:44 +0000 Subject: [PATCH 32/33] replaced sqlalchemy's dialect.name with flavor --- pandas/tests/io/test_sql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index cc6177779e380..0524a9c623216 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -1822,7 +1822,7 @@ def test_to_sql_with_negative_npinf(self, input): df = pd.DataFrame(input) - if self.conn.dialect.name == "mysql": + if self.conn.flavor == "mysql": msg = "inf cannot be used with MySQL" with pytest.raises(ValueError, match=msg): df.to_sql("foobar", self.conn, index=False) From 3b7c162fcc7becba1eb0c49d257376257d510fb9 Mon Sep 17 00:00:00 2001 From: arw2019 Date: Wed, 24 Jun 2020 19:44:15 +0000 Subject: [PATCH 33/33] fixed typo in test_to_sql-with-npinf --- pandas/tests/io/test_sql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 0524a9c623216..6d0e879aa19a0 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -1822,7 +1822,7 @@ def test_to_sql_with_negative_npinf(self, input): df = pd.DataFrame(input) - if self.conn.flavor == "mysql": + if self.flavor == "mysql": msg = "inf cannot be used with MySQL" with pytest.raises(ValueError, match=msg): df.to_sql("foobar", self.conn, index=False)