From cc105911c7956b85a79479e821da21232f470b05 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Mon, 5 Apr 2021 17:19:33 -0500 Subject: [PATCH 01/19] activate warnings --- .github/workflows/database.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/database.yml b/.github/workflows/database.yml index ba5a0a1fd0909..42077b04cc355 100644 --- a/.github/workflows/database.yml +++ b/.github/workflows/database.yml @@ -13,6 +13,7 @@ env: PANDAS_CI: 1 PATTERN: ((not slow and not network and not clipboard) or (single and db)) COVERAGE: true + SQLALCHEMY_WARN_20: true jobs: Linux_py37_IO: From 4c57caa9bcd3fcc2320927e4c1474794f2f2e261 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Sat, 10 Jul 2021 21:55:23 -0500 Subject: [PATCH 02/19] fix warnings --- pandas/tests/io/test_sql.py | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 290e063a59be7..afdc45032f51a 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -25,6 +25,7 @@ ) from io import StringIO import sqlite3 +from sqlite3.dbapi2 import Connection import warnings import numpy as np @@ -66,6 +67,7 @@ from sqlalchemy.orm import session as sa_session import sqlalchemy.schema import sqlalchemy.sql.sqltypes as sqltypes + from sqlalchemy import text SQLALCHEMY_INSTALLED = True except ImportError: @@ -288,10 +290,11 @@ class PandasSQLTest: """ def _get_exec(self): - if hasattr(self.conn, "execute"): - return self.conn - else: - return self.conn.cursor() + return ( + self.conn.cursor() + if isinstance(self.conn, sqlite3.Connection) + else self.conn + ) @pytest.fixture(params=[("io", "data", "csv", "iris.csv")]) def load_iris_data(self, datapath, request): @@ -302,15 +305,27 @@ def load_iris_data(self, datapath, request): self.setup_connect() self.drop_table("iris") - self._get_exec().execute(SQL_STRINGS["create_iris"][self.flavor]) + + if isinstance(self._get_exec(), sqlite3.Connection): + self._get_exec().execute(SQL_STRINGS["create_iris"][self.flavor]) + else: + with self._get_exec().connect() as conn: + with conn.begin(): + conn.execute(text(SQL_STRINGS["create_iris"][self.flavor])) with open(iris_csv_file, newline=None) as iris_csv: - r = csv.reader(iris_csv) - next(r) # skip header row + reader = csv.reader(iris_csv) + next(reader) # skip header row ins = SQL_STRINGS["insert_iris"][self.flavor] - for row in r: - self._get_exec().execute(ins, row) + if isinstance(self._get_exec(), sqlite3.Connection): + for row in reader: + self._get_exec().execute(ins, row) + else: + with self._get_exec().connect() as conn: + with conn.begin(): + for row in reader: + conn.execute(text(ins), row) def _load_iris_view(self): self.drop_table("iris_view") From 07d6ee489eb6d8764b5ff4606e86020e9d1a9f52 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Sat, 10 Jul 2021 22:19:47 -0500 Subject: [PATCH 03/19] fix warnings --- pandas/tests/io/test_sql.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index afdc45032f51a..721162e26d43a 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -306,7 +306,7 @@ def load_iris_data(self, datapath, request): self.drop_table("iris") - if isinstance(self._get_exec(), sqlite3.Connection): + if isinstance(self._get_exec(), sqlite3.Cursor): self._get_exec().execute(SQL_STRINGS["create_iris"][self.flavor]) else: with self._get_exec().connect() as conn: @@ -318,7 +318,7 @@ def load_iris_data(self, datapath, request): next(reader) # skip header row ins = SQL_STRINGS["insert_iris"][self.flavor] - if isinstance(self._get_exec(), sqlite3.Connection): + if isinstance(self._get_exec(), sqlite3.Cursor): for row in reader: self._get_exec().execute(ins, row) else: From c88ec69b071c007fa45dce81359f86b4377c8712 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Sat, 10 Jul 2021 23:24:29 -0500 Subject: [PATCH 04/19] remove text --- pandas/tests/io/test_sql.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 721162e26d43a..6fc3bb13f9976 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -67,7 +67,6 @@ from sqlalchemy.orm import session as sa_session import sqlalchemy.schema import sqlalchemy.sql.sqltypes as sqltypes - from sqlalchemy import text SQLALCHEMY_INSTALLED = True except ImportError: @@ -311,7 +310,7 @@ def load_iris_data(self, datapath, request): else: with self._get_exec().connect() as conn: with conn.begin(): - conn.execute(text(SQL_STRINGS["create_iris"][self.flavor])) + conn.execute(SQL_STRINGS["create_iris"][self.flavor]) with open(iris_csv_file, newline=None) as iris_csv: reader = csv.reader(iris_csv) @@ -325,7 +324,7 @@ def load_iris_data(self, datapath, request): with self._get_exec().connect() as conn: with conn.begin(): for row in reader: - conn.execute(text(ins), row) + conn.execute(ins, row) def _load_iris_view(self): self.drop_table("iris_view") From 6ad9aa69286935a0af9cd430bde2948367c45c99 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Sun, 11 Jul 2021 10:47:08 -0500 Subject: [PATCH 05/19] fix more warnings in tests --- pandas/tests/io/test_sql.py | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 6fc3bb13f9976..4f0685ea26338 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -328,7 +328,12 @@ def load_iris_data(self, datapath, request): def _load_iris_view(self): self.drop_table("iris_view") - self._get_exec().execute(SQL_STRINGS["create_view"][self.flavor]) + if isinstance(self._get_exec(), sqlite3.Cursor): + self._get_exec().execute(SQL_STRINGS["create_view"][self.flavor]) + else: + with self._get_exec().connect() as conn: + with conn.begin(): + conn.execute(SQL_STRINGS["create_view"][self.flavor]) def _check_iris_loaded_frame(self, iris_frame): pytype = iris_frame.dtypes[0].type @@ -447,7 +452,12 @@ def _filter_to_flavor(flavor, df): def _load_raw_sql(self): self.drop_table("types_test_data") - self._get_exec().execute(SQL_STRINGS["create_test_types"][self.flavor]) + if isinstance(self._get_exec(), sqlite3.Cursor): + self._get_exec().execute(SQL_STRINGS["create_test_types"][self.flavor]) + else: + with self._get_exec().connect() as conn: + with conn.begin(): + conn.execute(SQL_STRINGS["create_test_types"][self.flavor]) ins = SQL_STRINGS["insert_test_types"][self.flavor] data = [ { @@ -475,11 +485,18 @@ def _load_raw_sql(self): "BoolColWithNull": None, }, ] - - for d in data: - self._get_exec().execute( - ins["query"], [d[field] for field in ins["fields"]] - ) + if isinstance(self._get_exec(), sqlite3.Cursor): + for d in data: + self._get_exec().execute( + ins["query"], [d[field] for field in ins["fields"]] + ) + else: + with self._get_exec().connect() as conn: + with conn.begin(): + for d in data: + conn.execute( + ins["query"], [d[field] for field in ins["fields"]] + ) self._load_types_test_data(data) From 9aa8fb05ed1771ccfccaf03b7e37134d811d89ce Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Mon, 12 Jul 2021 23:19:32 -0500 Subject: [PATCH 06/19] fix more errors --- pandas/io/sql.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/pandas/io/sql.py b/pandas/io/sql.py index 1a8012df5cb46..ee56f2191db83 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -89,8 +89,12 @@ def _gt14() -> bool: return Version(sqlalchemy.__version__) >= Version("1.4.0") -def _convert_params(sql, params): +def _convert_params(sql, params, using_sqlite=False): """Convert SQL and params args to DBAPI2.0 compliant format.""" + if isinstance(sql, str) and not using_sqlite: + from sqlalchemy import text + + sql = text(sql) args = [sql] if params is not None: if hasattr(params, "keys"): # test if params is a mapping @@ -202,11 +206,15 @@ def execute(sql, con, cur=None, params=None): ------- Results Iterable """ + import sqlite3 + if cur is None: pandas_sql = pandasSQL_builder(con) else: pandas_sql = pandasSQL_builder(cur, is_cursor=True) - args = _convert_params(sql, params) + args = _convert_params( + sql, params, using_sqlite=isinstance(con, sqlite3.Connection) + ) return pandas_sql.execute(*args) @@ -1421,7 +1429,7 @@ def run_transaction(self): def execute(self, *args, **kwargs): """Simple passthrough to SQLAlchemy connectable""" - return self.connectable.execution_options().execute(*args, **kwargs) + return self.connectable.connect().execute(*args, **kwargs) def read_table( self, @@ -2112,7 +2120,7 @@ def read_query( dtype: DtypeArg | None = None, ): - args = _convert_params(sql, params) + args = _convert_params(sql, params, using_sqlite=True) cursor = self.execute(*args) columns = [col_desc[0] for col_desc in cursor.description] From b9fb23d26b2812f17d9f31743b6c9bca4de01122 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Tue, 13 Jul 2021 13:39:40 -0500 Subject: [PATCH 07/19] revert changes in tests --- pandas/io/sql.py | 2 ++ pandas/tests/io/test_sql.py | 52 ++++++++----------------------------- 2 files changed, 13 insertions(+), 41 deletions(-) diff --git a/pandas/io/sql.py b/pandas/io/sql.py index f5c349837b6fa..c204d995c4cb3 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -205,6 +205,8 @@ def execute(sql, con, params=None): ------- Results Iterable """ + import sqlite3 + pandas_sql = pandasSQL_builder(con) args = _convert_params( sql, params, using_sqlite=isinstance(con, sqlite3.Connection) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 37d3a2395315a..a54f51068d289 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -25,7 +25,6 @@ ) from io import StringIO import sqlite3 -from sqlite3.dbapi2 import Connection import warnings import numpy as np @@ -305,35 +304,19 @@ def load_iris_data(self, datapath, request): self.drop_table("iris") - if isinstance(self._get_exec(), sqlite3.Cursor): - self._get_exec().execute(SQL_STRINGS["create_iris"][self.flavor]) - else: - with self._get_exec().connect() as conn: - with conn.begin(): - conn.execute(SQL_STRINGS["create_iris"][self.flavor]) + self._get_exec().execute(SQL_STRINGS["create_iris"][self.flavor]) with open(iris_csv_file, newline=None) as iris_csv: - reader = csv.reader(iris_csv) - next(reader) # skip header row + r = csv.reader(iris_csv) + next(r) # skip header row ins = SQL_STRINGS["insert_iris"][self.flavor] - if isinstance(self._get_exec(), sqlite3.Cursor): - for row in reader: - self._get_exec().execute(ins, row) - else: - with self._get_exec().connect() as conn: - with conn.begin(): - for row in reader: - conn.execute(ins, row) + for row in r: + self._get_exec().execute(ins, row) def _load_iris_view(self): self.drop_table("iris_view") - if isinstance(self._get_exec(), sqlite3.Cursor): - self._get_exec().execute(SQL_STRINGS["create_view"][self.flavor]) - else: - with self._get_exec().connect() as conn: - with conn.begin(): - conn.execute(SQL_STRINGS["create_view"][self.flavor]) + self._get_exec().execute(SQL_STRINGS["create_view"][self.flavor]) def _check_iris_loaded_frame(self, iris_frame): pytype = iris_frame.dtypes[0].type @@ -452,12 +435,7 @@ def _filter_to_flavor(flavor, df): def _load_raw_sql(self): self.drop_table("types_test_data") - if isinstance(self._get_exec(), sqlite3.Cursor): - self._get_exec().execute(SQL_STRINGS["create_test_types"][self.flavor]) - else: - with self._get_exec().connect() as conn: - with conn.begin(): - conn.execute(SQL_STRINGS["create_test_types"][self.flavor]) + self._get_exec().execute(SQL_STRINGS["create_test_types"][self.flavor]) ins = SQL_STRINGS["insert_test_types"][self.flavor] data = [ { @@ -485,18 +463,10 @@ def _load_raw_sql(self): "BoolColWithNull": None, }, ] - if isinstance(self._get_exec(), sqlite3.Cursor): - for d in data: - self._get_exec().execute( - ins["query"], [d[field] for field in ins["fields"]] - ) - else: - with self._get_exec().connect() as conn: - with conn.begin(): - for d in data: - conn.execute( - ins["query"], [d[field] for field in ins["fields"]] - ) + for d in data: + self._get_exec().execute( + ins["query"], [d[field] for field in ins["fields"]] + ) self._load_types_test_data(data) From d768cdc1e0a8c1e7c4c0b08bbcca7af5eb4edd67 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Wed, 14 Jul 2021 14:05:13 -0500 Subject: [PATCH 08/19] update insert --- pandas/io/sql.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pandas/io/sql.py b/pandas/io/sql.py index c204d995c4cb3..063f41e0e9cf7 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -889,8 +889,11 @@ def _execute_insert_multi(self, conn, keys: list[str], data_iter): and tables containing a few columns but performance degrades quickly with increase of columns. """ + from sqlalchemy import insert + data = [dict(zip(keys, row)) for row in data_iter] - conn.execute(self.table.insert(data)) + insert_stmt = insert(self.table).values(data) + conn.execute(insert_stmt) def insert_data(self): if self.index is not None: @@ -1401,8 +1404,9 @@ class SQLDatabase(PandasSQL): arguments in the MetaData object. """ + from sqlalchemy import Engine - def __init__(self, engine, schema: str | None = None, meta=None): + def __init__(self, engine: Engine, schema: str | None = None, meta=None): self.connectable = engine if not meta: from sqlalchemy.schema import MetaData From 9d4f03be95ef51e17f1603a283c7cdbe18bde1ff Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Tue, 17 Aug 2021 16:46:00 -0500 Subject: [PATCH 09/19] revert changes --- pandas/io/sql.py | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/pandas/io/sql.py b/pandas/io/sql.py index 2161f122b1a03..afd045bd8bb2b 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -65,12 +65,8 @@ def _gt14() -> bool: return Version(sqlalchemy.__version__) >= Version("1.4.0") -def _convert_params(sql, params, using_sqlite=False): +def _convert_params(sql, params): """Convert SQL and params args to DBAPI2.0 compliant format.""" - if isinstance(sql, str) and not using_sqlite: - from sqlalchemy import text - - sql = text(sql) args = [sql] if params is not None: if hasattr(params, "keys"): # test if params is a mapping @@ -181,12 +177,8 @@ def execute(sql, con, params=None): ------- Results Iterable """ - import sqlite3 - pandas_sql = pandasSQL_builder(con) - args = _convert_params( - sql, params, using_sqlite=isinstance(con, sqlite3.Connection) - ) + args = _convert_params(sql, params) return pandas_sql.execute(*args) @@ -850,11 +842,8 @@ def _execute_insert_multi(self, conn, keys: list[str], data_iter): and tables containing a few columns but performance degrades quickly with increase of columns. """ - from sqlalchemy import insert - data = [dict(zip(keys, row)) for row in data_iter] - insert_stmt = insert(self.table).values(data) - conn.execute(insert_stmt) + conn.execute(self.table.insert(data)) def insert_data(self): if self.index is not None: @@ -1361,7 +1350,6 @@ class SQLDatabase(PandasSQL): supports this). If None, use default schema (default). """ - from sqlalchemy import Engine def __init__(self, engine, schema: str | None = None): from sqlalchemy.schema import MetaData @@ -1379,7 +1367,7 @@ def run_transaction(self): def execute(self, *args, **kwargs): """Simple passthrough to SQLAlchemy connectable""" - return self.connectable.connect().execute(*args, **kwargs) + return self.connectable.execution_options().execute(*args, **kwargs) def read_table( self, @@ -2066,7 +2054,7 @@ def read_query( dtype: DtypeArg | None = None, ): - args = _convert_params(sql, params, using_sqlite=True) + args = _convert_params(sql, params) cursor = self.execute(*args) columns = [col_desc[0] for col_desc in cursor.description] From c123658d2bee75f0055ab52114edacfa24ecf062 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Tue, 17 Aug 2021 16:47:18 -0500 Subject: [PATCH 10/19] revert changes --- pandas/tests/io/test_sql.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 009a9ad5e985a..6136bd0e1e057 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -416,11 +416,10 @@ class PandasSQLTest: """ def _get_exec(self): - return ( - self.conn.cursor() - if isinstance(self.conn, sqlite3.Connection) - else self.conn - ) + if hasattr(self.conn, "execute"): + return self.conn + else: + return self.conn.cursor() @pytest.fixture def load_iris_data(self, iris_path): From 9064a9848e1638209ae3f680ee98beef62e8b39e Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Wed, 18 Aug 2021 11:02:09 -0500 Subject: [PATCH 11/19] TST: resolve most of future warnnings in SQL test --- pandas/tests/io/test_sql.py | 173 ++++++++++++++++++++++-------------- 1 file changed, 108 insertions(+), 65 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 6136bd0e1e057..48186cb063e2f 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -63,11 +63,6 @@ try: import sqlalchemy - from sqlalchemy import inspect - from sqlalchemy.ext import declarative - from sqlalchemy.orm import session as sa_session - import sqlalchemy.schema - import sqlalchemy.sql.sqltypes as sqltypes SQLALCHEMY_INSTALLED = True except ImportError: @@ -399,10 +394,9 @@ def drop_table(self, table_name): sql.SQLDatabase(self.conn).drop_table(table_name) def _get_all_tables(self): - meta = sqlalchemy.schema.MetaData(bind=self.conn) - meta.reflect() - table_list = meta.tables.keys() - return table_list + from sqlalchemy import inspect + + return inspect(self.conn).get_table_names() def _close_conn(self): # https://docs.sqlalchemy.org/en/13/core/connections.html#engine-disposal @@ -1123,10 +1117,12 @@ def test_read_sql_delegate(self): tm.assert_frame_equal(iris_frame1, iris_frame2) def test_not_reflect_all_tables(self): + from sqlalchemy import text + # create invalid table - qry = """CREATE TABLE invalid (x INTEGER, y UNKNOWN);""" + qry = text("CREATE TABLE invalid (x INTEGER, y UNKNOWN);") self.conn.execute(qry) - qry = """CREATE TABLE other_table (x INTEGER, y INTEGER);""" + qry = text("CREATE TABLE other_table (x INTEGER, y INTEGER);") self.conn.execute(qry) with warnings.catch_warnings(record=True) as w: @@ -1161,6 +1157,7 @@ def _get_index_columns(self, tbl_name): return ixs def test_sqlalchemy_type_mapping(self): + from sqlalchemy import TIMESTAMP # Test Timestamp objects (no datetime64 because of timezone) (GH9085) df = DataFrame( @@ -1169,7 +1166,7 @@ def test_sqlalchemy_type_mapping(self): db = sql.SQLDatabase(self.conn) table = sql.SQLTable("test_type", db, frame=df) # GH 9086: TIMESTAMP is the suggested type for datetimes with timezones - assert isinstance(table.table.c["time"].type, sqltypes.TIMESTAMP) + assert isinstance(table.table.c["time"].type, TIMESTAMP) @pytest.mark.parametrize( "integer, expected", @@ -1252,7 +1249,7 @@ def test_query_by_select_obj(self): # WIP : GH10846 iris = iris_table_metadata(self.flavor) - name_select = sqlalchemy.select([iris]).where( + name_select = sqlalchemy.select(iris).where( iris.c.Name == sqlalchemy.bindparam("name") ) iris_df = sql.read_sql(name_select, self.conn, params={"name": "Iris-setosa"}) @@ -1464,6 +1461,8 @@ def test_create_table(self): pandasSQL.to_sql(temp_frame, "temp_frame") if _gt14(): + from sqlalchemy import inspect + insp = inspect(temp_conn) assert insp.has_table("temp_frame") else: @@ -1480,6 +1479,8 @@ def test_drop_table(self): pandasSQL.to_sql(temp_frame, "temp_frame") if _gt14(): + from sqlalchemy import inspect + insp = inspect(temp_conn) assert insp.has_table("temp_frame") else: @@ -1842,48 +1843,68 @@ def test_get_schema_create_table(self, test_frame3): # Use a dataframe without a bool column, since MySQL converts bool to # TINYINT (which read_sql_table returns as an int and causes a dtype # mismatch) + from sqlalchemy import text + from sqlalchemy.engine import Engine tbl = "test_get_schema_create_table" create_sql = sql.get_schema(test_frame3, tbl, con=self.conn) blank_test_df = test_frame3.iloc[:0] self.drop_table(tbl) - self.conn.execute(create_sql) + create_sql = text(create_sql) + if isinstance(self.conn, Engine): + with self.conn.connect() as conn: + conn.execute(create_sql) + else: + self.conn.execute(create_sql) returned_df = sql.read_sql_table(tbl, self.conn) tm.assert_frame_equal(returned_df, blank_test_df, check_index_type=False) self.drop_table(tbl) def test_dtype(self): + from sqlalchemy import ( + TEXT, + String, + ) + from sqlalchemy.schema import MetaData + cols = ["A", "B"] data = [(0.8, True), (0.9, None)] df = DataFrame(data, columns=cols) df.to_sql("dtype_test", self.conn) - df.to_sql("dtype_test2", self.conn, dtype={"B": sqlalchemy.TEXT}) - meta = sqlalchemy.schema.MetaData(bind=self.conn) - meta.reflect() + df.to_sql("dtype_test2", self.conn, dtype={"B": TEXT}) + meta = MetaData() + meta.reflect(bind=self.conn) sqltype = meta.tables["dtype_test2"].columns["B"].type - assert isinstance(sqltype, sqlalchemy.TEXT) + assert isinstance(sqltype, TEXT) msg = "The type of B is not a SQLAlchemy type" with pytest.raises(ValueError, match=msg): df.to_sql("error", self.conn, dtype={"B": str}) # GH9083 - df.to_sql("dtype_test3", self.conn, dtype={"B": sqlalchemy.String(10)}) - meta.reflect() + df.to_sql("dtype_test3", self.conn, dtype={"B": String(10)}) + meta.reflect(bind=self.conn) sqltype = meta.tables["dtype_test3"].columns["B"].type - assert isinstance(sqltype, sqlalchemy.String) + assert isinstance(sqltype, String) assert sqltype.length == 10 # single dtype - df.to_sql("single_dtype_test", self.conn, dtype=sqlalchemy.TEXT) - meta = sqlalchemy.schema.MetaData(bind=self.conn) - meta.reflect() + df.to_sql("single_dtype_test", self.conn, dtype=TEXT) + meta.reflect(bind=self.conn) sqltypea = meta.tables["single_dtype_test"].columns["A"].type sqltypeb = meta.tables["single_dtype_test"].columns["B"].type - assert isinstance(sqltypea, sqlalchemy.TEXT) - assert isinstance(sqltypeb, sqlalchemy.TEXT) + assert isinstance(sqltypea, TEXT) + assert isinstance(sqltypeb, TEXT) def test_notna_dtype(self): + from sqlalchemy import ( + Boolean, + DateTime, + Float, + Integer, + ) + from sqlalchemy.schema import MetaData + cols = { "Bool": Series([True, None]), "Date": Series([datetime(2012, 5, 1), None]), @@ -1894,22 +1915,24 @@ def test_notna_dtype(self): tbl = "notna_dtype_test" df.to_sql(tbl, self.conn) - returned_df = sql.read_sql_table(tbl, self.conn) # noqa - meta = sqlalchemy.schema.MetaData(bind=self.conn) - meta.reflect() - if self.flavor == "mysql": - my_type = sqltypes.Integer - else: - my_type = sqltypes.Boolean - + _ = sql.read_sql_table(tbl, self.conn) + meta = MetaData() + meta.reflect(bind=self.conn) + my_type = Integer if self.flavor == "mysql" else Boolean col_dict = meta.tables[tbl].columns - assert isinstance(col_dict["Bool"].type, my_type) - assert isinstance(col_dict["Date"].type, sqltypes.DateTime) - assert isinstance(col_dict["Int"].type, sqltypes.Integer) - assert isinstance(col_dict["Float"].type, sqltypes.Float) + assert isinstance(col_dict["Date"].type, DateTime) + assert isinstance(col_dict["Int"].type, Integer) + assert isinstance(col_dict["Float"].type, Float) def test_double_precision(self): + from sqlalchemy import ( + BigInteger, + Float, + Integer, + ) + from sqlalchemy.schema import MetaData + V = 1.23456789101112131415 df = DataFrame( @@ -1927,7 +1950,7 @@ def test_double_precision(self): self.conn, index=False, if_exists="replace", - dtype={"f64_as_f32": sqlalchemy.Float(precision=23)}, + dtype={"f64_as_f32": Float(precision=23)}, ) res = sql.read_sql_table("test_dtypes", self.conn) @@ -1935,18 +1958,19 @@ def test_double_precision(self): assert np.round(df["f64"].iloc[0], 14) == np.round(res["f64"].iloc[0], 14) # check sql types - meta = sqlalchemy.schema.MetaData(bind=self.conn) - meta.reflect() + meta = MetaData() + meta.reflect(bind=self.conn) col_dict = meta.tables["test_dtypes"].columns assert str(col_dict["f32"].type) == str(col_dict["f64_as_f32"].type) - assert isinstance(col_dict["f32"].type, sqltypes.Float) - assert isinstance(col_dict["f64"].type, sqltypes.Float) - assert isinstance(col_dict["i32"].type, sqltypes.Integer) - assert isinstance(col_dict["i64"].type, sqltypes.BigInteger) + assert isinstance(col_dict["f32"].type, Float) + assert isinstance(col_dict["f64"].type, Float) + assert isinstance(col_dict["i32"].type, Integer) + assert isinstance(col_dict["i64"].type, BigInteger) def test_connectable_issue_example(self): # This tests the example raised in issue # https://github.com/pandas-dev/pandas/issues/10104 + from sqlalchemy.engine import Engine def foo(connection): query = "SELECT test_foo_data FROM test_foo_data" @@ -1955,17 +1979,23 @@ def foo(connection): def bar(connection, data): data.to_sql(name="test_foo_data", con=connection, if_exists="append") + def baz(conn): + if _gt14(): + # https://github.com/sqlalchemy/sqlalchemy/commit/ + # 00b5c10846e800304caa86549ab9da373b42fa5d#r48323973 + foo_data = foo(conn) + bar(conn, foo_data) + else: + foo_data = conn.run_callable(foo) + conn.run_callable(bar, foo_data) + def main(connectable): - with connectable.connect() as conn: - with conn.begin(): - if _gt14(): - # https://github.com/sqlalchemy/sqlalchemy/commit/ - # 00b5c10846e800304caa86549ab9da373b42fa5d#r48323973 - foo_data = foo(conn) - bar(conn, foo_data) - else: - foo_data = conn.run_callable(foo) - conn.run_callable(bar, foo_data) + if isinstance(connectable, Engine): + with connectable.connect() as conn: + with conn.begin(): + baz(conn) + else: + baz(connectable) DataFrame({"test_foo_data": [0, 1, 2]}).to_sql("test_foo_data", self.conn) main(self.conn) @@ -1999,24 +2029,35 @@ def test_to_sql_with_negative_npinf(self, input, request): tm.assert_equal(df, res) def test_temporary_table(self): + from sqlalchemy import ( + Column, + Integer, + Unicode, + select, + ) + from sqlalchemy.orm import ( + declarative_base, + sessionmaker, + ) + test_data = "Hello, World!" expected = DataFrame({"spam": [test_data]}) - Base = declarative.declarative_base() + Base = declarative_base() class Temporary(Base): __tablename__ = "temp_test" __table_args__ = {"prefixes": ["TEMPORARY"]} - id = sqlalchemy.Column(sqlalchemy.Integer, primary_key=True) - spam = sqlalchemy.Column(sqlalchemy.Unicode(30), nullable=False) + id = Column(Integer, primary_key=True) + spam = Column(Unicode(30), nullable=False) - Session = sa_session.sessionmaker(bind=self.conn) + Session = sessionmaker(bind=self.conn) session = Session() - with session.transaction: + with session.begin(): conn = session.connection() Temporary.__table__.create(conn) session.add(Temporary(spam=test_data)) session.flush() - df = sql.read_sql_query(sql=sqlalchemy.select([Temporary.spam]), con=conn) + df = sql.read_sql_query(sql=select(Temporary.spam), con=conn) tm.assert_frame_equal(df, expected) @@ -2120,7 +2161,7 @@ class _TestMySQLAlchemy: @classmethod def connect(cls): return sqlalchemy.create_engine( - f"mysql+{cls.driver}://root@localhost:{cls.port}/pandas", + f"mysql+{cls.driver}://root:cdma1993@localhost:{cls.port}/pandas", connect_args=cls.connect_args, ) @@ -2147,6 +2188,8 @@ def test_default_type_conversion(self): def test_read_procedure(self): import pymysql + from sqlalchemy import text + from sqlalchemy.engine import Engine # see GH7324. Although it is more an api test, it is added to the # mysql tests as sqlite does not have stored procedures @@ -2160,11 +2203,11 @@ def test_read_procedure(self): BEGIN SELECT * FROM test_procedure; END""" - - connection = self.conn.connect() + proc = text(proc) + connection = self.conn.connect() if isinstance(self.conn, Engine) else self.conn trans = connection.begin() try: - r1 = connection.execute(proc) # noqa + _ = connection.execute(proc) trans.commit() except pymysql.Error: trans.rollback() From 1a449bee437fe29f1473800a772f761502517819 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Wed, 18 Aug 2021 11:09:08 -0500 Subject: [PATCH 12/19] fix import --- pandas/tests/io/test_sql.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 48186cb063e2f..5dfa52834f1ef 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -981,8 +981,10 @@ def test_get_schema_with_schema(self, test_frame1): assert "CREATE TABLE pypi." in create_sql def test_get_schema_dtypes(self): + from sqlalchemy import Integer + float_frame = DataFrame({"a": [1.1, 1.2], "b": [2.1, 2.2]}) - dtype = sqlalchemy.Integer if self.mode == "sqlalchemy" else "INTEGER" + dtype = Integer if self.mode == "sqlalchemy" else "INTEGER" create_sql = sql.get_schema( float_frame, "test", con=self.conn, dtype={"b": dtype} ) @@ -1240,18 +1242,23 @@ def test_database_uri_string(self, test_frame1): def test_query_by_text_obj(self): # WIP : GH10846 - name_text = sqlalchemy.text("select * from iris where name=:name") + from sqlalchemy import text + + name_text = text("select * from iris where name=:name") iris_df = sql.read_sql(name_text, self.conn, params={"name": "Iris-versicolor"}) all_names = set(iris_df["Name"]) assert all_names == {"Iris-versicolor"} def test_query_by_select_obj(self): # WIP : GH10846 + from sqlalchemy import ( + bindparam, + select, + ) + iris = iris_table_metadata(self.flavor) - name_select = sqlalchemy.select(iris).where( - iris.c.Name == sqlalchemy.bindparam("name") - ) + name_select = select(iris).where(iris.c.Name == bindparam("name")) iris_df = sql.read_sql(name_select, self.conn, params={"name": "Iris-setosa"}) all_names = set(iris_df["Name"]) assert all_names == {"Iris-setosa"} @@ -2242,6 +2249,8 @@ def setup_driver(cls): cls.driver = "psycopg2" def test_schema_support(self): + from sqlalchemy.engine import Engine + # only test this for postgresql (schema's not supported in # mysql/sqlite) df = DataFrame({"col1": [1, 2], "col2": [0.1, 0.2], "col3": ["a", "n"]}) @@ -2301,7 +2310,7 @@ def test_schema_support(self): # The schema won't be applied on another Connection # because of transactional schemas - if isinstance(self.conn, sqlalchemy.engine.Engine): + if isinstance(self.conn, Engine): engine2 = self.connect() pdsql = sql.SQLDatabase(engine2, schema="other") pdsql.to_sql(df, "test_schema_other2", index=False) From dd9a8d9f954f2fc70f58a49f57154912ae5783a4 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Wed, 18 Aug 2021 11:12:27 -0500 Subject: [PATCH 13/19] turn of warnings --- .github/workflows/database.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/database.yml b/.github/workflows/database.yml index a431480d34d29..cedc2b85c2794 100644 --- a/.github/workflows/database.yml +++ b/.github/workflows/database.yml @@ -16,7 +16,6 @@ env: PANDAS_CI: 1 PATTERN: ((not slow and not network and not clipboard) or (single and db)) COVERAGE: true - SQLALCHEMY_WARN_20: true jobs: Linux_py38_IO: From 379856163742adc4ec70076bbc7e3a0be0f60ce8 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Wed, 18 Aug 2021 11:15:23 -0500 Subject: [PATCH 14/19] change conn url back --- pandas/tests/io/test_sql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 5dfa52834f1ef..69480badeef16 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -2168,7 +2168,7 @@ class _TestMySQLAlchemy: @classmethod def connect(cls): return sqlalchemy.create_engine( - f"mysql+{cls.driver}://root:cdma1993@localhost:{cls.port}/pandas", + f"mysql+{cls.driver}://root@localhost:{cls.port}/pandas", connect_args=cls.connect_args, ) From a1925e3b4dcc45d6d62837892b3ffca54306352e Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Wed, 18 Aug 2021 13:15:29 -0500 Subject: [PATCH 15/19] fix import --- pandas/tests/io/test_sql.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 69480badeef16..68ab06bb1c913 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -981,10 +981,14 @@ def test_get_schema_with_schema(self, test_frame1): assert "CREATE TABLE pypi." in create_sql def test_get_schema_dtypes(self): - from sqlalchemy import Integer + if self.mode == "sqlalchemy": + from sqlalchemy import Integer + + dtype = Integer + else: + dtype = "INTEGER" float_frame = DataFrame({"a": [1.1, 1.2], "b": [2.1, 2.2]}) - dtype = Integer if self.mode == "sqlalchemy" else "INTEGER" create_sql = sql.get_schema( float_frame, "test", con=self.conn, dtype={"b": dtype} ) @@ -2042,10 +2046,12 @@ def test_temporary_table(self): Unicode, select, ) - from sqlalchemy.orm import ( - declarative_base, - sessionmaker, - ) + from sqlalchemy.orm import sessionmaker + + if _gt14(): + from sqlalchemy.orm import declarative_base + else: + from sqlalchemy.ext.declarative import declarative_base test_data = "Hello, World!" expected = DataFrame({"spam": [test_data]}) From e0b5a1ddceaf504d30ea03981ce39923732a5664 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Wed, 18 Aug 2021 17:31:57 -0500 Subject: [PATCH 16/19] session compat --- pandas/tests/io/test_sql.py | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 68ab06bb1c913..5829f80a94ae4 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -1261,8 +1261,8 @@ def test_query_by_select_obj(self): ) iris = iris_table_metadata(self.flavor) - - name_select = select(iris).where(iris.c.Name == bindparam("name")) + iris_select = iris if _gt14() else [iris] + name_select = select(iris_select).where(iris.c.Name == bindparam("name")) iris_df = sql.read_sql(name_select, self.conn, params={"name": "Iris-setosa"}) all_names = set(iris_df["Name"]) assert all_names == {"Iris-setosa"} @@ -2046,7 +2046,7 @@ def test_temporary_table(self): Unicode, select, ) - from sqlalchemy.orm import sessionmaker + from sqlalchemy.orm import Session if _gt14(): from sqlalchemy.orm import declarative_base @@ -2063,14 +2063,22 @@ class Temporary(Base): id = Column(Integer, primary_key=True) spam = Column(Unicode(30), nullable=False) - Session = sessionmaker(bind=self.conn) - session = Session() - with session.begin(): - conn = session.connection() - Temporary.__table__.create(conn) - session.add(Temporary(spam=test_data)) - session.flush() - df = sql.read_sql_query(sql=select(Temporary.spam), con=conn) + if _gt14(): + with Session(self.conn) as session: + with session.begin(): + conn = session.connection() + Temporary.__table__.create(conn) + session.add(Temporary(spam=test_data)) + session.flush() + df = sql.read_sql_query(sql=select(Temporary.spam), con=conn) + else: + with Session(self.conn) as session: + with session.transaction: + conn = session.connection() + Temporary.__table__.create(conn) + session.add(Temporary(spam=test_data)) + session.flush() + df = sql.read_sql_query(sql=select([Temporary.spam]), con=conn) tm.assert_frame_equal(df, expected) @@ -2174,7 +2182,7 @@ class _TestMySQLAlchemy: @classmethod def connect(cls): return sqlalchemy.create_engine( - f"mysql+{cls.driver}://root@localhost:{cls.port}/pandas", + f"mysql+{cls.driver}://root:cdma1993@localhost:{cls.port}/pandas", connect_args=cls.connect_args, ) From 66aa480ac5e177e064823a420ac02109fcf7c108 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Wed, 18 Aug 2021 17:39:30 -0500 Subject: [PATCH 17/19] change conn url back --- pandas/tests/io/test_sql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 5829f80a94ae4..4bd21a4c0b112 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -2182,7 +2182,7 @@ class _TestMySQLAlchemy: @classmethod def connect(cls): return sqlalchemy.create_engine( - f"mysql+{cls.driver}://root:cdma1993@localhost:{cls.port}/pandas", + f"mysql+{cls.driver}://root@localhost:{cls.port}/pandas", connect_args=cls.connect_args, ) From 0dad34967f5fc3a733c47ebec800fb07f9c5c10a Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Wed, 18 Aug 2021 18:56:43 -0500 Subject: [PATCH 18/19] fix sqlalchemy 1.3 compat --- pandas/tests/io/test_sql.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 4bd21a4c0b112..e1251aeb127bf 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -2046,7 +2046,7 @@ def test_temporary_table(self): Unicode, select, ) - from sqlalchemy.orm import Session + from sqlalchemy.orm import Session, sessionmaker if _gt14(): from sqlalchemy.orm import declarative_base @@ -2072,13 +2072,14 @@ class Temporary(Base): session.flush() df = sql.read_sql_query(sql=select(Temporary.spam), con=conn) else: - with Session(self.conn) as session: - with session.transaction: - conn = session.connection() - Temporary.__table__.create(conn) - session.add(Temporary(spam=test_data)) - session.flush() - df = sql.read_sql_query(sql=select([Temporary.spam]), con=conn) + Session = sessionmaker() + session = Session(bind=self.conn) + with session.transaction: + conn = session.connection() + Temporary.__table__.create(conn) + session.add(Temporary(spam=test_data)) + session.flush() + df = sql.read_sql_query(sql=select([Temporary.spam]), con=conn) tm.assert_frame_equal(df, expected) From df2bbadaa866527eda12b993ba4e070244645598 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Wed, 18 Aug 2021 19:05:40 -0500 Subject: [PATCH 19/19] format import --- pandas/tests/io/test_sql.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index e1251aeb127bf..14a35f4ae6460 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -2046,7 +2046,10 @@ def test_temporary_table(self): Unicode, select, ) - from sqlalchemy.orm import Session, sessionmaker + from sqlalchemy.orm import ( + Session, + sessionmaker, + ) if _gt14(): from sqlalchemy.orm import declarative_base