From 40a2d2cb9a568eec961ddf29502d456d6f06b13c Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Wed, 14 Jul 2021 18:33:34 -0500 Subject: [PATCH 1/7] CLN: clean sqlalchemy import --- pandas/io/sql.py | 87 +++++++++++++----------------------------------- 1 file changed, 24 insertions(+), 63 deletions(-) diff --git a/pandas/io/sql.py b/pandas/io/sql.py index df9c7e28bff69..25540d7087f34 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -61,24 +61,6 @@ class DatabaseError(IOError): _SQLALCHEMY_INSTALLED: bool | None = None -def _is_sqlalchemy_connectable(con): - global _SQLALCHEMY_INSTALLED - if _SQLALCHEMY_INSTALLED is None: - try: - import sqlalchemy - - _SQLALCHEMY_INSTALLED = True - except ImportError: - _SQLALCHEMY_INSTALLED = False - - if _SQLALCHEMY_INSTALLED: - import sqlalchemy # noqa: F811 - - return isinstance(con, sqlalchemy.engine.Connectable) - else: - return False - - def _gt14() -> bool: """ Check if sqlalchemy.__version__ is at least 1.4.0, when several @@ -303,21 +285,7 @@ def read_sql_table( -------- >>> pd.read_sql_table('table_name', 'postgres:///db_name') # doctest:+SKIP """ - con = _engine_builder(con) - if not _is_sqlalchemy_connectable(con): - raise NotImplementedError( - "read_sql_table only supported for SQLAlchemy connectable." - ) - import sqlalchemy - from sqlalchemy.schema import MetaData - - meta = MetaData(con, schema=schema) - try: - meta.reflect(only=[table_name], views=True) - except sqlalchemy.exc.InvalidRequestError as err: - raise ValueError(f"Table {table_name} not found") from err - - pandas_sql = SQLDatabase(con, meta=meta) + pandas_sql = pandasSQL_builder(con, schema=schema, table_name=table_name) table = pandas_sql.read_table( table_name, index_col=index_col, @@ -752,37 +720,25 @@ def has_table(table_name: str, con, schema: str | None = None): table_exists = has_table -def _engine_builder(con): - """ - Returns a SQLAlchemy engine from a URI (if con is a string) - else it just return con without modifying it. - """ - global _SQLALCHEMY_INSTALLED - if isinstance(con, str): - try: - import sqlalchemy - except ImportError: - _SQLALCHEMY_INSTALLED = False - else: - con = sqlalchemy.create_engine(con) - return con - - return con - - -def pandasSQL_builder(con, schema: str | None = None, meta=None): +def pandasSQL_builder(con, schema: str | None = None, table_name=None): """ Convenience function to return the correct PandasSQL subclass based on the provided parameters. """ - con = _engine_builder(con) - if _is_sqlalchemy_connectable(con): - return SQLDatabase(con, schema=schema, meta=meta) - elif isinstance(con, str): - raise ImportError("Using URI string without sqlalchemy installed.") - else: + import sqlite3 + + if isinstance(con, sqlite3.Connection) or con is None: return SQLiteDatabase(con) + msg = "sqlalchemy is not installed" + sqlalchemy = import_optional_dependency("sqlalchemy", extra=msg) + + if isinstance(con, str): + con = sqlalchemy.create_engine(con) + + if isinstance(con, sqlalchemy.engine.Connectable): + return SQLDatabase(con, schema=schema, table_name=table_name) + class SQLTable(PandasObject): """ @@ -1394,14 +1350,19 @@ class SQLDatabase(PandasSQL): """ - def __init__(self, engine, schema: str | None = None, meta=None): + def __init__(self, engine, schema: str | None = None, table_name=None): self.connectable = engine - if not meta: - from sqlalchemy.schema import MetaData - meta = MetaData(self.connectable, schema=schema) + from sqlalchemy.schema import MetaData + + self.meta = MetaData(self.connectable, schema=schema) + if table_name is not None: + from sqlalchemy.exc import InvalidRequestError - self.meta = meta + try: + self.meta.reflect(only=[table_name], views=True) + except InvalidRequestError as err: + raise ValueError(f"Table {table_name} not found") from err @contextmanager def run_transaction(self): From 3dfb9cf6d77becb658fd8cb3bb74bad30e5f55af Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Wed, 14 Jul 2021 19:44:25 -0500 Subject: [PATCH 2/7] update test --- pandas/io/sql.py | 4 +++- pandas/tests/io/test_sql.py | 3 +-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pandas/io/sql.py b/pandas/io/sql.py index 25540d7087f34..9cd9bf9c2f126 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -1350,7 +1350,9 @@ class SQLDatabase(PandasSQL): """ - def __init__(self, engine, schema: str | None = None, table_name=None): + def __init__( + self, engine, schema: str | None = None, table_name: str | None = None + ): self.connectable = engine from sqlalchemy.schema import MetaData diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 9320bf385ce0a..70d292c04d04d 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -2314,8 +2314,7 @@ def test_schema_support(self): # because of transactional schemas if isinstance(self.conn, sqlalchemy.engine.Engine): engine2 = self.connect() - meta = sqlalchemy.MetaData(engine2, schema="other") - pdsql = sql.SQLDatabase(engine2, meta=meta) + pdsql = sql.SQLDatabase(engine2, schema="other") pdsql.to_sql(df, "test_schema_other2", index=False) pdsql.to_sql(df, "test_schema_other2", index=False, if_exists="replace") pdsql.to_sql(df, "test_schema_other2", index=False, if_exists="append") From 7b92dbbc87261e41fdbb06ea9851936c6f70b57d Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Thu, 15 Jul 2021 15:09:52 -0500 Subject: [PATCH 3/7] update test --- pandas/io/sql.py | 4 +--- pandas/tests/io/test_sql.py | 3 +-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/pandas/io/sql.py b/pandas/io/sql.py index 9cd9bf9c2f126..dc76e40478f4d 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -58,8 +58,6 @@ class DatabaseError(IOError): # ----------------------------------------------------------------------------- # -- Helper functions -_SQLALCHEMY_INSTALLED: bool | None = None - def _gt14() -> bool: """ @@ -720,7 +718,7 @@ def has_table(table_name: str, con, schema: str | None = None): table_exists = has_table -def pandasSQL_builder(con, schema: str | None = None, table_name=None): +def pandasSQL_builder(con, schema: str | None = None, table_name: str | None = None): """ Convenience function to return the correct PandasSQL subclass based on the provided parameters. diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 70d292c04d04d..bfc82ee32ec69 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -1372,8 +1372,7 @@ def test_sql_open_close(self): @pytest.mark.skipif(SQLALCHEMY_INSTALLED, reason="SQLAlchemy is installed") def test_con_string_import_error(self): conn = "mysql://root@localhost/pandas" - msg = "Using URI string without sqlalchemy installed" - with pytest.raises(ImportError, match=msg): + with pytest.raises(ImportError): sql.read_sql("SELECT * FROM iris", conn) def test_read_sql_delegate(self): From 647a4c148f10d56647adef4462803a87ecc88ea2 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Thu, 15 Jul 2021 23:06:58 -0500 Subject: [PATCH 4/7] add msg to test, refactor --- pandas/io/sql.py | 35 ++++++++++++----------------------- pandas/tests/io/test_sql.py | 2 +- 2 files changed, 13 insertions(+), 24 deletions(-) diff --git a/pandas/io/sql.py b/pandas/io/sql.py index dc76e40478f4d..10f34e29f50e6 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -47,10 +47,6 @@ from pandas.util.version import Version -class SQLAlchemyRequired(ImportError): - pass - - class DatabaseError(IOError): pass @@ -283,7 +279,14 @@ def read_sql_table( -------- >>> pd.read_sql_table('table_name', 'postgres:///db_name') # doctest:+SKIP """ - pandas_sql = pandasSQL_builder(con, schema=schema, table_name=table_name) + from sqlalchemy.exc import InvalidRequestError + + pandas_sql = pandasSQL_builder(con, schema=schema) + try: + pandas_sql.meta.reflect(only=[table_name], views=True) + except InvalidRequestError as err: + raise ValueError(f"Table {table_name} not found") from err + table = pandas_sql.read_table( table_name, index_col=index_col, @@ -718,7 +721,7 @@ def has_table(table_name: str, con, schema: str | None = None): table_exists = has_table -def pandasSQL_builder(con, schema: str | None = None, table_name: str | None = None): +def pandasSQL_builder(con, schema: str | None = None): """ Convenience function to return the correct PandasSQL subclass based on the provided parameters. @@ -735,7 +738,7 @@ def pandasSQL_builder(con, schema: str | None = None, table_name: str | None = N con = sqlalchemy.create_engine(con) if isinstance(con, sqlalchemy.engine.Connectable): - return SQLDatabase(con, schema=schema, table_name=table_name) + return SQLDatabase(con, schema=schema) class SQLTable(PandasObject): @@ -1341,28 +1344,14 @@ class SQLDatabase(PandasSQL): schema : string, default None Name of SQL schema in database to write to (if database flavor supports this). If None, use default schema (default). - meta : SQLAlchemy MetaData object, default None - If provided, this MetaData object is used instead of a newly - created. This allows to specify database flavor specific - arguments in the MetaData object. """ - def __init__( - self, engine, schema: str | None = None, table_name: str | None = None - ): - self.connectable = engine - + def __init__(self, engine, schema: str | None = None): from sqlalchemy.schema import MetaData + self.connectable = engine self.meta = MetaData(self.connectable, schema=schema) - if table_name is not None: - from sqlalchemy.exc import InvalidRequestError - - try: - self.meta.reflect(only=[table_name], views=True) - except InvalidRequestError as err: - raise ValueError(f"Table {table_name} not found") from err @contextmanager def run_transaction(self): diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index bfc82ee32ec69..626b36acd8de4 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -1372,7 +1372,7 @@ def test_sql_open_close(self): @pytest.mark.skipif(SQLALCHEMY_INSTALLED, reason="SQLAlchemy is installed") def test_con_string_import_error(self): conn = "mysql://root@localhost/pandas" - with pytest.raises(ImportError): + with pytest.raises(ImportError, match="sqlalchemy"): sql.read_sql("SELECT * FROM iris", conn) def test_read_sql_delegate(self): From 7db6f6247e31194caa38518ba6ad52f6cfd6df78 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Thu, 15 Jul 2021 23:42:40 -0500 Subject: [PATCH 5/7] raise ValueError for incorrect input type --- pandas/io/sql.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pandas/io/sql.py b/pandas/io/sql.py index 10f34e29f50e6..32dbaaf985417 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -19,6 +19,7 @@ Sequence, cast, overload, + TYPE_CHECKING, ) import warnings @@ -731,8 +732,7 @@ def pandasSQL_builder(con, schema: str | None = None): if isinstance(con, sqlite3.Connection) or con is None: return SQLiteDatabase(con) - msg = "sqlalchemy is not installed" - sqlalchemy = import_optional_dependency("sqlalchemy", extra=msg) + sqlalchemy = import_optional_dependency("sqlalchemy") if isinstance(con, str): con = sqlalchemy.create_engine(con) @@ -740,6 +740,11 @@ def pandasSQL_builder(con, schema: str | None = None): if isinstance(con, sqlalchemy.engine.Connectable): return SQLDatabase(con, schema=schema) + raise ValueError( + "pandas only support SQLAlchemy connectable(engine/connection) or" + "database string URI or sqlite3 DBAPI2 connection" + ) + class SQLTable(PandasObject): """ From 2f1162312ee4caefd47c1386c3fdd485d95e57cc Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Thu, 15 Jul 2021 23:50:39 -0500 Subject: [PATCH 6/7] fix import --- pandas/io/sql.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pandas/io/sql.py b/pandas/io/sql.py index 32dbaaf985417..afd045bd8bb2b 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -19,7 +19,6 @@ Sequence, cast, overload, - TYPE_CHECKING, ) import warnings From 0f880186cd6fedd129ec2fe1b45c07859590b424 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Fri, 16 Jul 2021 19:46:01 -0500 Subject: [PATCH 7/7] fix error msg in test --- pandas/tests/io/test_sql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 626b36acd8de4..df9ba5f206146 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -1372,7 +1372,7 @@ def test_sql_open_close(self): @pytest.mark.skipif(SQLALCHEMY_INSTALLED, reason="SQLAlchemy is installed") def test_con_string_import_error(self): conn = "mysql://root@localhost/pandas" - with pytest.raises(ImportError, match="sqlalchemy"): + with pytest.raises(ImportError, match="SQLAlchemy"): sql.read_sql("SELECT * FROM iris", conn) def test_read_sql_delegate(self):