From 76d95f7ff06fb853ff6786dfc03116bd44fd77a8 Mon Sep 17 00:00:00 2001 From: Patrick Hoefler Date: Thu, 5 Jan 2023 09:39:03 +0100 Subject: [PATCH 01/15] CI: Try with sqlalchemy warnings --- ci/run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/run_tests.sh b/ci/run_tests.sh index a48d6c1ad6580..05252f1e93e37 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -24,7 +24,7 @@ if [[ $(uname) == "Linux" && -z $DISPLAY ]]; then XVFB="xvfb-run " fi -PYTEST_CMD="${XVFB}pytest -r fEs -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET" +PYTEST_CMD="${XVFB}SQLALCHEMY_WARN_20=1 pytest -r fEs -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET" if [[ "$PATTERN" ]]; then PYTEST_CMD="$PYTEST_CMD -m \"$PATTERN\"" From c135a64a8dcf2339b934c126b19eac6aaf174d39 Mon Sep 17 00:00:00 2001 From: Patrick Hoefler Date: Thu, 5 Jan 2023 09:58:52 +0100 Subject: [PATCH 02/15] CI: Try with sqlalchemy warnings --- ci/run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/run_tests.sh b/ci/run_tests.sh index 05252f1e93e37..921c82e548193 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -24,7 +24,7 @@ if [[ $(uname) == "Linux" && -z $DISPLAY ]]; then XVFB="xvfb-run " fi -PYTEST_CMD="${XVFB}SQLALCHEMY_WARN_20=1 pytest -r fEs -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET" +PYTEST_CMD="SQLALCHEMY_WARN_20=1 ${XVFB}pytest -r fEs -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET" if [[ "$PATTERN" ]]; then PYTEST_CMD="$PYTEST_CMD -m \"$PATTERN\"" From 06c1ae54060d48e3b1d5d7dfb9cf81f4b00d8f67 Mon Sep 17 00:00:00 2001 From: Patrick Hoefler Date: Thu, 5 Jan 2023 12:47:39 +0100 Subject: [PATCH 03/15] Use text --- pandas/tests/io/test_sql.py | 99 ++++++++++++++++++++++--------------- 1 file changed, 60 insertions(+), 39 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 31ca060e36ad1..ae70fc96876f7 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -71,34 +71,40 @@ try: import sqlalchemy + from sqlalchemy import text SQLALCHEMY_INSTALLED = True except ImportError: SQLALCHEMY_INSTALLED = False + text = lambda x: x SQL_STRINGS = { "read_parameters": { "sqlite": "SELECT * FROM iris WHERE Name=? AND SepalLength=?", - "mysql": "SELECT * FROM iris WHERE `Name`=%s AND `SepalLength`=%s", - "postgresql": 'SELECT * FROM iris WHERE "Name"=%s AND "SepalLength"=%s', + "mysql": text("SELECT * FROM iris WHERE `Name`=%s AND `SepalLength`=%s"), + "postgresql": text('SELECT * FROM iris WHERE "Name"=%s AND "SepalLength"=%s'), }, "read_named_parameters": { "sqlite": """ SELECT * FROM iris WHERE Name=:name AND SepalLength=:length """, - "mysql": """ + "mysql": text( + """ SELECT * FROM iris WHERE `Name`=%(name)s AND `SepalLength`=%(length)s - """, - "postgresql": """ + """ + ), + "postgresql": text( + """ SELECT * FROM iris WHERE "Name"=%(name)s AND "SepalLength"=%(length)s - """, + """ + ), }, "read_no_parameters_with_percent": { "sqlite": "SELECT * FROM iris WHERE Name LIKE '%'", - "mysql": "SELECT * FROM iris WHERE `Name` LIKE '%'", - "postgresql": "SELECT * FROM iris WHERE \"Name\" LIKE '%'", + "mysql": text("SELECT * FROM iris WHERE `Name` LIKE '%'"), + "postgresql": text("SELECT * FROM iris WHERE \"Name\" LIKE '%'"), }, } @@ -544,7 +550,10 @@ def test_to_sql_exist_fail(conn, test_frame1, request): def test_read_iris(conn, request): conn = request.getfixturevalue(conn) with pandasSQL_builder(conn) as pandasSQL: - iris_frame = pandasSQL.read_query("SELECT * FROM iris") + if isinstance(conn, sqlite3.Connection): + iris_frame = pandasSQL.read_query("SELECT * FROM iris") + else: + iris_frame = pandasSQL.read_query(text("SELECT * FROM iris")) check_iris_frame(iris_frame) @@ -665,7 +674,7 @@ def psql_insert_copy(table, conn, keys, data_iter): def test_execute_typeerror(sqlite_iris_engine): with pytest.raises(TypeError, match="pandas.io.sql.execute requires a connection"): - sql.execute("select * from iris", sqlite_iris_engine) + sql.execute(text("select * from iris"), sqlite_iris_engine) class MixInBase: @@ -721,6 +730,8 @@ class PandasSQLTest: """ + text = lambda x, y: text(y) + def load_iris_data(self, iris_path): self.drop_table("iris", self.conn) if isinstance(self.conn, sqlite3.Connection): @@ -782,7 +793,9 @@ def _to_sql_with_sql_engine(self, test_frame1, engine="auto", **engine_kwargs): def _roundtrip(self, test_frame1): self.drop_table("test_frame_roundtrip", self.conn) assert self.pandasSQL.to_sql(test_frame1, "test_frame_roundtrip") == 4 - result = self.pandasSQL.read_query("SELECT * FROM test_frame_roundtrip") + result = self.pandasSQL.read_query( + self.text("SELECT * FROM test_frame_roundtrip") + ) result.set_index("level_0", inplace=True) # result.index.astype(int) @@ -793,7 +806,7 @@ def _roundtrip(self, test_frame1): def _execute_sql(self): # drop_sql = "DROP TABLE IF EXISTS test" # should already be done - iris_results = self.pandasSQL.execute("SELECT * FROM iris") + iris_results = self.pandasSQL.execute(self.text("SELECT * FROM iris")) row = iris_results.fetchone() tm.equalContents(row, [5.1, 3.5, 1.4, 0.2, "Iris-setosa"]) @@ -832,13 +845,13 @@ class DummyException(Exception): except DummyException: # ignore raised exception pass - res = self.pandasSQL.read_query("SELECT * FROM test_trans") + res = self.pandasSQL.read_query(self.text("SELECT * FROM test_trans")) assert len(res) == 0 # Make sure when transaction is committed, rows do get inserted with self.pandasSQL.run_transaction() as trans: trans.execute(ins_sql) - res2 = self.pandasSQL.read_query("SELECT * FROM test_trans") + res2 = self.pandasSQL.read_query(self.text("SELECT * FROM test_trans")) assert len(res2) == 1 @@ -879,11 +892,11 @@ def load_test_data_and_sql(self): create_and_load_iris_view(self.conn) def test_read_sql_view(self): - iris_frame = sql.read_sql_query("SELECT * FROM iris_view", self.conn) + iris_frame = sql.read_sql_query(self.text("SELECT * FROM iris_view"), self.conn) check_iris_frame(iris_frame) def test_read_sql_with_chunksize_no_result(self): - query = "SELECT * FROM iris_view WHERE SepalLength < 0.0" + query = self.text("SELECT * FROM iris_view WHERE SepalLength < 0.0") with_batch = sql.read_sql_query(query, self.conn, chunksize=5) without_batch = sql.read_sql_query(query, self.conn) tm.assert_frame_equal(concat(with_batch), without_batch) @@ -927,19 +940,21 @@ def test_to_sql_append(self, test_frame1): def test_to_sql_type_mapping(self, test_frame3): sql.to_sql(test_frame3, "test_frame5", self.conn, index=False) - result = sql.read_sql("SELECT * FROM test_frame5", self.conn) + result = sql.read_sql(self.text("SELECT * FROM test_frame5"), self.conn) tm.assert_frame_equal(test_frame3, result) def test_to_sql_series(self): s = Series(np.arange(5, dtype="int64"), name="series") sql.to_sql(s, "test_series", self.conn, index=False) - s2 = sql.read_sql_query("SELECT * FROM test_series", self.conn) + s2 = sql.read_sql_query(self.text("SELECT * FROM test_series"), self.conn) tm.assert_frame_equal(s.to_frame(), s2) def test_roundtrip(self, test_frame1): sql.to_sql(test_frame1, "test_frame_roundtrip", con=self.conn) - result = sql.read_sql_query("SELECT * FROM test_frame_roundtrip", con=self.conn) + result = sql.read_sql_query( + self.text("SELECT * FROM test_frame_roundtrip"), con=self.conn + ) # HACK! result.index = test_frame1.index @@ -956,24 +971,26 @@ def test_roundtrip_chunksize(self, test_frame1): index=False, chunksize=2, ) - result = sql.read_sql_query("SELECT * FROM test_frame_roundtrip", con=self.conn) + result = sql.read_sql_query( + self.text("SELECT * FROM test_frame_roundtrip"), con=self.conn + ) tm.assert_frame_equal(result, test_frame1) def test_execute_sql(self): # drop_sql = "DROP TABLE IF EXISTS test" # should already be done with sql.pandasSQL_builder(self.conn) as pandas_sql: - iris_results = pandas_sql.execute("SELECT * FROM iris") + iris_results = pandas_sql.execute(self.text("SELECT * FROM iris")) row = iris_results.fetchone() tm.equalContents(row, [5.1, 3.5, 1.4, 0.2, "Iris-setosa"]) def test_date_parsing(self): # Test date parsing in read_sql # No Parsing - df = sql.read_sql_query("SELECT * FROM types", self.conn) + df = sql.read_sql_query(self.text("SELECT * FROM types"), self.conn) assert not issubclass(df.DateCol.dtype.type, np.datetime64) df = sql.read_sql_query( - "SELECT * FROM types", self.conn, parse_dates=["DateCol"] + self.text("SELECT * FROM types"), self.conn, parse_dates=["DateCol"] ) assert issubclass(df.DateCol.dtype.type, np.datetime64) assert df.DateCol.tolist() == [ @@ -982,7 +999,7 @@ def test_date_parsing(self): ] df = sql.read_sql_query( - "SELECT * FROM types", + self.text("SELECT * FROM types"), self.conn, parse_dates={"DateCol": "%Y-%m-%d %H:%M:%S"}, ) @@ -993,7 +1010,7 @@ def test_date_parsing(self): ] df = sql.read_sql_query( - "SELECT * FROM types", self.conn, parse_dates=["IntDateCol"] + self.text("SELECT * FROM types"), self.conn, parse_dates=["IntDateCol"] ) assert issubclass(df.IntDateCol.dtype.type, np.datetime64) assert df.IntDateCol.tolist() == [ @@ -1002,7 +1019,7 @@ def test_date_parsing(self): ] df = sql.read_sql_query( - "SELECT * FROM types", self.conn, parse_dates={"IntDateCol": "s"} + self.text("SELECT * FROM types"), self.conn, parse_dates={"IntDateCol": "s"} ) assert issubclass(df.IntDateCol.dtype.type, np.datetime64) assert df.IntDateCol.tolist() == [ @@ -1011,7 +1028,7 @@ def test_date_parsing(self): ] df = sql.read_sql_query( - "SELECT * FROM types", + self.text("SELECT * FROM types"), self.conn, parse_dates={"IntDateOnlyCol": "%Y%m%d"}, ) @@ -1023,7 +1040,7 @@ def test_date_parsing(self): @pytest.mark.parametrize("error", ["ignore", "raise", "coerce"]) @pytest.mark.parametrize( - "read_sql, text, mode", + "read_sql, query, mode", [ (sql.read_sql, "SELECT * FROM types", ("sqlalchemy", "fallback")), (sql.read_sql, "types", ("sqlalchemy")), @@ -1036,13 +1053,13 @@ def test_date_parsing(self): ], ) def test_custom_dateparsing_error( - self, read_sql, text, mode, error, types_data_frame + self, read_sql, query, mode, error, types_data_frame ): if self.mode in mode: expected = types_data_frame.astype({"DateCol": "datetime64[ns]"}) result = read_sql( - text, + query, con=self.conn, parse_dates={ "DateCol": {"errors": error}, @@ -1055,7 +1072,7 @@ def test_date_and_index(self): # Test case where same column appears in parse_date and index_col df = sql.read_sql_query( - "SELECT * FROM types", + self.text("SELECT * FROM types"), self.conn, index_col="DateCol", parse_dates=["DateCol", "IntDateCol"], @@ -1071,7 +1088,9 @@ def test_timedelta(self): with tm.assert_produces_warning(UserWarning): result_count = df.to_sql("test_timedelta", self.conn) assert result_count == 2 - result = sql.read_sql_query("SELECT * FROM test_timedelta", self.conn) + result = sql.read_sql_query( + self.text("SELECT * FROM test_timedelta"), self.conn + ) tm.assert_series_equal(result["foo"], df["foo"].view("int64")) def test_complex_raises(self): @@ -1240,7 +1259,7 @@ def test_chunksize_read(self): df.to_sql("test_chunksize", self.conn, index=False) # reading the query in one time - res1 = sql.read_sql_query("select * from test_chunksize", self.conn) + res1 = sql.read_sql_query(self.text("select * from test_chunksize"), self.conn) # reading the query in chunks with read_sql_query res2 = DataFrame() @@ -1248,7 +1267,7 @@ def test_chunksize_read(self): sizes = [5, 5, 5, 5, 2] for chunk in sql.read_sql_query( - "select * from test_chunksize", self.conn, chunksize=5 + self.text("select * from test_chunksize"), self.conn, chunksize=5 ): res2 = concat([res2, chunk], ignore_index=True) assert len(chunk) == sizes[i] @@ -1470,7 +1489,7 @@ def test_pg8000_sqlalchemy_passthrough_error(self): # in sqlalchemy.create_engine -> test passing of this error to user db_uri = "postgresql+pg8000://user:pass@host/dbname" with pytest.raises(ImportError, match="pg8000"): - sql.read_sql("select * from table", db_uri) + sql.read_sql(text("select * from table"), db_uri) def test_query_by_text_obj(self): # WIP : GH10846 @@ -1512,6 +1531,7 @@ class TestSQLiteFallbackApi(SQLiteMixIn, _TestSQLApi): flavor = "sqlite" mode = "fallback" + text = lambda x, y: y def connect(self, database=":memory:"): return sqlite3.connect(database) @@ -1756,7 +1776,7 @@ def check(col): ) # GH11216 - df = read_sql_query("select * from types", self.conn) + df = read_sql_query(text("select * from types"), self.conn) if not hasattr(df, "DateColWithTz"): request.node.add_marker( pytest.mark.xfail(reason="no column with datetime with time zone") @@ -1769,7 +1789,7 @@ def check(col): assert is_datetime64tz_dtype(col.dtype) df = read_sql_query( - "select * from types", self.conn, parse_dates=["DateColWithTz"] + text("select * from types"), self.conn, parse_dates=["DateColWithTz"] ) if not hasattr(df, "DateColWithTz"): request.node.add_marker( @@ -1781,7 +1801,7 @@ def check(col): check(df.DateColWithTz) df = concat( - list(read_sql_query("select * from types", self.conn, chunksize=1)), + list(read_sql_query(text("select * from types"), self.conn, chunksize=1)), ignore_index=True, ) col = df.DateColWithTz @@ -2158,7 +2178,7 @@ def test_connectable_issue_example(self): from sqlalchemy.engine import Engine def test_select(connection): - query = "SELECT test_foo_data FROM test_foo_data" + query = text("SELECT test_foo_data FROM test_foo_data") return sql.read_sql_query(query, con=connection) def test_append(connection, data): @@ -2622,6 +2642,7 @@ class TestSQLiteFallback(SQLiteMixIn, PandasSQLTest): """ flavor = "sqlite" + text = lambda x, y: y @pytest.fixture(autouse=True) def setup_method(self, iris_path, types_data): From ce44a3c0ba8139e2f4da6a4ceeabc440cd5f3d80 Mon Sep 17 00:00:00 2001 From: Patrick Hoefler Date: Thu, 5 Jan 2023 14:40:20 +0100 Subject: [PATCH 04/15] Remove other warnings --- pandas/tests/io/test_sql.py | 66 +++++++++++++++++++++++-------------- 1 file changed, 41 insertions(+), 25 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index ae70fc96876f7..93a4635f87b02 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -750,13 +750,13 @@ def load_types_data(self, types_data): create_and_load_types(self.conn, types_data, self.flavor) def _read_sql_iris_parameter(self): - query = SQL_STRINGS["read_parameters"][self.flavor] + query = self.text(SQL_STRINGS["read_parameters"][self.flavor]) params = ["Iris-setosa", 5.1] iris_frame = self.pandasSQL.read_query(query, params=params) check_iris_frame(iris_frame) def _read_sql_iris_named_parameter(self): - query = SQL_STRINGS["read_named_parameters"][self.flavor] + query = self.text(SQL_STRINGS["read_named_parameters"][self.flavor]) params = {"name": "Iris-setosa", "length": 5.1} iris_frame = self.pandasSQL.read_query(query, params=params) check_iris_frame(iris_frame) @@ -1059,7 +1059,7 @@ def test_custom_dateparsing_error( expected = types_data_frame.astype({"DateCol": "datetime64[ns]"}) result = read_sql( - query, + self.text(query), con=self.conn, parse_dates={ "DateCol": {"errors": error}, @@ -1119,7 +1119,7 @@ def test_complex_raises(self): def test_to_sql_index_label(self, index_name, index_label, expected): temp_frame = DataFrame({"col1": range(4)}) temp_frame.index.name = index_name - query = "SELECT * FROM test_index_label" + query = self.text("SELECT * FROM test_index_label") sql.to_sql(temp_frame, "test_index_label", self.conn, index_label=index_label) frame = sql.read_sql_query(query, self.conn) assert frame.columns[0] == expected @@ -1134,7 +1134,9 @@ def test_to_sql_index_label_multiindex(self): # no index name, defaults to 'level_0' and 'level_1' result = sql.to_sql(temp_frame, "test_index_label", self.conn) assert result == expected_row_count - frame = sql.read_sql_query("SELECT * FROM test_index_label", self.conn) + frame = sql.read_sql_query( + self.text("SELECT * FROM test_index_label"), self.conn + ) assert frame.columns[0] == "level_0" assert frame.columns[1] == "level_1" @@ -1147,7 +1149,9 @@ def test_to_sql_index_label_multiindex(self): index_label=["A", "B"], ) assert result == expected_row_count - frame = sql.read_sql_query("SELECT * FROM test_index_label", self.conn) + frame = sql.read_sql_query( + self.text("SELECT * FROM test_index_label"), self.conn + ) assert frame.columns[:2].tolist() == ["A", "B"] # using the index name @@ -1156,7 +1160,9 @@ def test_to_sql_index_label_multiindex(self): temp_frame, "test_index_label", self.conn, if_exists="replace" ) assert result == expected_row_count - frame = sql.read_sql_query("SELECT * FROM test_index_label", self.conn) + frame = sql.read_sql_query( + self.text("SELECT * FROM test_index_label"), self.conn + ) assert frame.columns[:2].tolist() == ["A", "B"] # has index name, but specifying index_label @@ -1168,7 +1174,9 @@ def test_to_sql_index_label_multiindex(self): index_label=["C", "D"], ) assert result == expected_row_count - frame = sql.read_sql_query("SELECT * FROM test_index_label", self.conn) + frame = sql.read_sql_query( + self.text("SELECT * FROM test_index_label"), self.conn + ) assert frame.columns[:2].tolist() == ["C", "D"] msg = "Length of 'index_label' should match number of levels, which is 2" @@ -1190,7 +1198,9 @@ def test_multiindex_roundtrip(self): df.to_sql("test_multiindex_roundtrip", self.conn) result = sql.read_sql_query( - "SELECT * FROM test_multiindex_roundtrip", self.conn, index_col=["A", "B"] + self.text("SELECT * FROM test_multiindex_roundtrip"), + self.conn, + index_col=["A", "B"], ) tm.assert_frame_equal(df, result, check_index_type=True) @@ -1210,7 +1220,9 @@ def test_dtype_argument(self, dtype): expected = df.astype(dtype) result = sql.read_sql_query( - "SELECT A, B FROM test_dtype_argument", con=self.conn, dtype=dtype + self.text("SELECT A, B FROM test_dtype_argument"), + con=self.conn, + dtype=dtype, ) tm.assert_frame_equal(result, expected) @@ -1301,7 +1313,7 @@ def test_categorical(self): df2["person_name"] = df2["person_name"].astype("category") df2.to_sql("test_categorical", self.conn, index=False) - res = sql.read_sql_query("SELECT * FROM test_categorical", self.conn) + res = sql.read_sql_query(self.text("SELECT * FROM test_categorical"), self.conn) tm.assert_frame_equal(res, df) @@ -1315,7 +1327,9 @@ def test_escaped_table_name(self): df = DataFrame({"A": [0, 1, 2], "B": [0.2, np.nan, 5.6]}) df.to_sql("d1187b08-4943-4c8d-a7f6", self.conn, index=False) - res = sql.read_sql_query("SELECT * FROM `d1187b08-4943-4c8d-a7f6`", self.conn) + res = sql.read_sql_query( + self.text("SELECT * FROM `d1187b08-4943-4c8d-a7f6`"), self.conn + ) tm.assert_frame_equal(res, df) @@ -1389,7 +1403,7 @@ def test_not_reflect_all_tables(self): with tm.assert_produces_warning(None): sql.read_sql_table("other_table", self.conn) - sql.read_sql_query("SELECT * FROM other_table", self.conn) + sql.read_sql_query(self.text("SELECT * FROM other_table"), self.conn) def test_warning_case_insensitive_table_name(self, test_frame1): # see gh-7815 @@ -1477,7 +1491,7 @@ def test_database_uri_string(self, test_frame1): test_frame1.to_sql(table, db_uri, if_exists="replace", index=False) test_frame2 = sql.read_sql(table, db_uri) test_frame3 = sql.read_sql_table(table, db_uri) - query = "SELECT * FROM iris" + query = self.text("SELECT * FROM iris") test_frame4 = sql.read_sql_query(query, db_uri) tm.assert_frame_equal(test_frame1, test_frame2) tm.assert_frame_equal(test_frame1, test_frame3) @@ -1579,8 +1593,8 @@ def close(self): sql.read_sql("SELECT 1", conn) def test_read_sql_delegate(self): - iris_frame1 = sql.read_sql_query("SELECT * FROM iris", self.conn) - iris_frame2 = sql.read_sql("SELECT * FROM iris", self.conn) + iris_frame1 = sql.read_sql_query(self.text("SELECT * FROM iris"), self.conn) + iris_frame2 = sql.read_sql(self.text("SELECT * FROM iris"), self.conn) tm.assert_frame_equal(iris_frame1, iris_frame2) msg = "Execution failed on sql 'iris': near \"iris\": syntax error" @@ -1837,7 +1851,9 @@ def test_datetime_with_timezone_roundtrip(self): result = sql.read_sql_table("test_datetime_tz", self.conn) tm.assert_frame_equal(result, expected) - result = sql.read_sql_query("SELECT * FROM test_datetime_tz", self.conn) + result = sql.read_sql_query( + self.text("SELECT * FROM test_datetime_tz"), self.conn + ) if self.flavor == "sqlite": # read_sql_query does not return datetime type like read_sql_table assert isinstance(result.loc[0, "A"], str) @@ -1906,7 +1922,7 @@ def test_datetime(self): tm.assert_frame_equal(result, df) # with read_sql -> no type information -> sqlite has no native - result = sql.read_sql_query("SELECT * FROM test_datetime", self.conn) + result = sql.read_sql_query(self.text("SELECT * FROM test_datetime"), self.conn) result = result.drop("index", axis=1) if self.flavor == "sqlite": assert isinstance(result.loc[0, "A"], str) @@ -1927,7 +1943,7 @@ def test_datetime_NaT(self): tm.assert_frame_equal(result, df) # with read_sql -> no type information -> sqlite has no native - result = sql.read_sql_query("SELECT * FROM test_datetime", self.conn) + result = sql.read_sql_query(self.text("SELECT * FROM test_datetime"), self.conn) if self.flavor == "sqlite": assert isinstance(result.loc[0, "A"], str) result["A"] = to_datetime(result["A"], errors="coerce") @@ -1990,7 +2006,7 @@ def test_nan_numeric(self): tm.assert_frame_equal(result, df) # with read_sql - result = sql.read_sql_query("SELECT * FROM test_nan", self.conn) + result = sql.read_sql_query(self.text("SELECT * FROM test_nan"), self.conn) tm.assert_frame_equal(result, df) def test_nan_fullcolumn(self): @@ -2005,7 +2021,7 @@ def test_nan_fullcolumn(self): # with read_sql -> not type info from table -> stays None df["B"] = df["B"].astype("object") df["B"] = None - result = sql.read_sql_query("SELECT * FROM test_nan", self.conn) + result = sql.read_sql_query(self.text("SELECT * FROM test_nan"), self.conn) tm.assert_frame_equal(result, df) def test_nan_string(self): @@ -2021,7 +2037,7 @@ def test_nan_string(self): tm.assert_frame_equal(result, df) # with read_sql - result = sql.read_sql_query("SELECT * FROM test_nan", self.conn) + result = sql.read_sql_query(self.text("SELECT * FROM test_nan"), self.conn) tm.assert_frame_equal(result, df) def _get_index_columns(self, tbl_name): @@ -2305,7 +2321,7 @@ def test_read_sql_nullable_dtypes(self, string_storage, func): with pd.option_context("mode.string_storage", string_storage): result = getattr(pd, func)( - f"Select * from {table}", self.conn, use_nullable_dtypes=True + self.text(f"Select * from {table}"), self.conn, use_nullable_dtypes=True ) expected = self.nullable_expected(string_storage) tm.assert_frame_equal(result, expected) @@ -2392,7 +2408,7 @@ def test_chunksize_empty_dtypes(self): df.to_sql("test", self.conn, index=False, if_exists="replace") for result in read_sql_query( - "SELECT * FROM test", + self.text("SELECT * FROM test"), self.conn, dtype=dtypes, chunksize=1, @@ -2702,7 +2718,7 @@ def test_datetime_time(self, tz_aware): df = DataFrame(tz_times, columns=["a"]) assert df.to_sql("test_time", self.conn, index=False) == 2 - res = read_sql_query("SELECT * FROM test_time", self.conn) + res = read_sql_query(self.text("SELECT * FROM test_time"), self.conn) if self.flavor == "sqlite": # comes back as strings expected = df.applymap(lambda _: _.strftime("%H:%M:%S.%f")) From 24de73fa4e1adf83b22a0f58314c5395e32aba5a Mon Sep 17 00:00:00 2001 From: Patrick Hoefler Date: Thu, 5 Jan 2023 14:42:43 +0100 Subject: [PATCH 05/15] Adjust lambda --- pandas/tests/io/test_sql.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 93a4635f87b02..85061cbec48a0 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -730,7 +730,7 @@ class PandasSQLTest: """ - text = lambda x, y: text(y) + text = lambda self, x: text(x) def load_iris_data(self, iris_path): self.drop_table("iris", self.conn) @@ -1545,7 +1545,7 @@ class TestSQLiteFallbackApi(SQLiteMixIn, _TestSQLApi): flavor = "sqlite" mode = "fallback" - text = lambda x, y: y + text = lambda self, x: x def connect(self, database=":memory:"): return sqlite3.connect(database) @@ -2658,7 +2658,7 @@ class TestSQLiteFallback(SQLiteMixIn, PandasSQLTest): """ flavor = "sqlite" - text = lambda x, y: y + text = lambda self, x: x @pytest.fixture(autouse=True) def setup_method(self, iris_path, types_data): From 37c52cf21d380b87145e13896733d3d6a8b99446 Mon Sep 17 00:00:00 2001 From: Patrick Hoefler Date: Thu, 5 Jan 2023 16:08:28 +0100 Subject: [PATCH 06/15] Fix tests --- pandas/tests/io/test_sql.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 85061cbec48a0..d936455c0d8d1 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -750,7 +750,8 @@ def load_types_data(self, types_data): create_and_load_types(self.conn, types_data, self.flavor) def _read_sql_iris_parameter(self): - query = self.text(SQL_STRINGS["read_parameters"][self.flavor]) + query = SQL_STRINGS["read_parameters"][self.flavor] + query = self.text(query) if isinstance(query, str) else query params = ["Iris-setosa", 5.1] iris_frame = self.pandasSQL.read_query(query, params=params) check_iris_frame(iris_frame) @@ -1055,11 +1056,12 @@ def test_date_parsing(self): def test_custom_dateparsing_error( self, read_sql, query, mode, error, types_data_frame ): + query = self.text(query) if isinstance(query, str) else query if self.mode in mode: expected = types_data_frame.astype({"DateCol": "datetime64[ns]"}) result = read_sql( - self.text(query), + query, con=self.conn, parse_dates={ "DateCol": {"errors": error}, @@ -1376,8 +1378,8 @@ def test_read_table_index_col(self, test_frame1): assert result.columns.tolist() == ["C", "D"] def test_read_sql_delegate(self): - iris_frame1 = sql.read_sql_query("SELECT * FROM iris", self.conn) - iris_frame2 = sql.read_sql("SELECT * FROM iris", self.conn) + iris_frame1 = sql.read_sql_query(self.text("SELECT * FROM iris"), self.conn) + iris_frame2 = sql.read_sql(self.text("SELECT * FROM iris"), self.conn) tm.assert_frame_equal(iris_frame1, iris_frame2) iris_frame1 = sql.read_sql_table("iris", self.conn) @@ -2328,7 +2330,7 @@ def test_read_sql_nullable_dtypes(self, string_storage, func): with pd.option_context("mode.string_storage", string_storage): iterator = getattr(pd, func)( - f"Select * from {table}", + self.text(f"Select * from {table}"), self.conn, use_nullable_dtypes=True, chunksize=3, From dbae45014f4e8369cbb517f9b87408ae681fc816 Mon Sep 17 00:00:00 2001 From: Patrick Hoefler Date: Thu, 5 Jan 2023 17:42:14 +0100 Subject: [PATCH 07/15] Fix tests --- pandas/tests/io/test_sql.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index d936455c0d8d1..e38d16b8c0d41 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -1056,7 +1056,9 @@ def test_date_parsing(self): def test_custom_dateparsing_error( self, read_sql, query, mode, error, types_data_frame ): - query = self.text(query) if isinstance(query, str) else query + query = ( + self.text(query) if isinstance(query, str) and query != "types" else query + ) if self.mode in mode: expected = types_data_frame.astype({"DateCol": "datetime64[ns]"}) @@ -2425,6 +2427,7 @@ class TestSQLiteAlchemy(_TestSQLAlchemy): """ flavor = "sqlite" + text = lambda self, x: x @classmethod def setup_engine(cls): From 0e37c08cf9b02cc3d275b6b0097b4bc5a1d12c6a Mon Sep 17 00:00:00 2001 From: Patrick Hoefler Date: Thu, 5 Jan 2023 18:03:14 +0100 Subject: [PATCH 08/15] Try begin --- pandas/io/sql.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/io/sql.py b/pandas/io/sql.py index 2b845786b0366..83971153ec2c1 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -902,7 +902,8 @@ def sql_schema(self) -> str: def _execute_create(self) -> None: # Inserting table into database, add to MetaData object self.table = self.table.to_metadata(self.pd_sql.meta) - self.table.create(bind=self.pd_sql.con) + with self.pd_sql.con.begin(): + self.table.create(bind=self.pd_sql.con) def create(self) -> None: if self.exists(): From 9ca89ba3602d10bf1a114c82eab840f72990e209 Mon Sep 17 00:00:00 2001 From: Patrick Hoefler Date: Thu, 5 Jan 2023 22:09:26 +0100 Subject: [PATCH 09/15] Fix tests --- pandas/io/sql.py | 3 +-- pandas/tests/io/test_sql.py | 2 ++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pandas/io/sql.py b/pandas/io/sql.py index 83971153ec2c1..2b845786b0366 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -902,8 +902,7 @@ def sql_schema(self) -> str: def _execute_create(self) -> None: # Inserting table into database, add to MetaData object self.table = self.table.to_metadata(self.pd_sql.meta) - with self.pd_sql.con.begin(): - self.table.create(bind=self.pd_sql.con) + self.table.create(bind=self.pd_sql.con) def create(self) -> None: if self.exists(): diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index e38d16b8c0d41..f90b5d35faebe 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -1682,6 +1682,7 @@ def test_create_table(self): from sqlalchemy import inspect temp_conn = self.connect() + temp_conn.begin() temp_frame = DataFrame( {"one": [1.0, 2.0, 3.0, 4.0], "two": [4.0, 3.0, 2.0, 1.0]} ) @@ -1695,6 +1696,7 @@ def test_drop_table(self): from sqlalchemy import inspect temp_conn = self.connect() + temp_conn.begin() temp_frame = DataFrame( {"one": [1.0, 2.0, 3.0, 4.0], "two": [4.0, 3.0, 2.0, 1.0]} ) From d99b4e79dc7134f5d6d0a7f6a06b6d0489927004 Mon Sep 17 00:00:00 2001 From: Patrick Hoefler Date: Thu, 5 Jan 2023 23:42:55 +0100 Subject: [PATCH 10/15] Fix test --- pandas/tests/io/test_sql.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index f90b5d35faebe..dde9aa35d1918 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -2429,7 +2429,6 @@ class TestSQLiteAlchemy(_TestSQLAlchemy): """ flavor = "sqlite" - text = lambda self, x: x @classmethod def setup_engine(cls): @@ -2440,6 +2439,10 @@ def setup_driver(cls): # sqlite3 is built-in cls.driver = None + @pytest.mark.xfail(reason="sorted on strings and integers") + def test_read_sql_parameter(self): + self._read_sql_iris_parameter() + def test_default_type_conversion(self): df = sql.read_sql_table("types", self.conn) From e0d421a68f7c96f7d59277f7d95285aa2c1d29fa Mon Sep 17 00:00:00 2001 From: Patrick Hoefler Date: Fri, 6 Jan 2023 08:41:44 +0100 Subject: [PATCH 11/15] Fix tests --- pandas/tests/io/test_sql.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index dde9aa35d1918..a816dca9d08be 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -1978,13 +1978,13 @@ def test_datetime_time(self, sqlite_buildin): # first, use the fallback to have the sqlite adapter put in place sqlite_conn = sqlite_buildin assert sql.to_sql(df, "test_time2", sqlite_conn, index=False) == 2 - res = sql.read_sql_query("SELECT * FROM test_time2", sqlite_conn) + res = sql.read_sql_query(self.text("SELECT * FROM test_time2"), sqlite_conn) ref = df.applymap(lambda _: _.strftime("%H:%M:%S.%f")) tm.assert_frame_equal(ref, res) # check if adapter is in place # then test if sqlalchemy is unaffected by the sqlite adapter assert sql.to_sql(df, "test_time3", self.conn, index=False) == 2 if self.flavor == "sqlite": - res = sql.read_sql_query("SELECT * FROM test_time3", self.conn) + res = sql.read_sql_query(self.text("SELECT * FROM test_time3"), self.conn) ref = df.applymap(lambda _: _.strftime("%H:%M:%S.%f")) tm.assert_frame_equal(ref, res) res = sql.read_sql_table("test_time3", self.conn) @@ -2439,7 +2439,7 @@ def setup_driver(cls): # sqlite3 is built-in cls.driver = None - @pytest.mark.xfail(reason="sorted on strings and integers") + @pytest.mark.skip(reason="sorted on strings and integers") def test_read_sql_parameter(self): self._read_sql_iris_parameter() From 4573f12d9e75416f9156ddbd07bd21b695495aad Mon Sep 17 00:00:00 2001 From: Patrick Hoefler Date: Fri, 6 Jan 2023 09:28:16 +0100 Subject: [PATCH 12/15] Fix --- pandas/tests/io/test_sql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index a816dca9d08be..ffac2b1d014ec 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -1978,7 +1978,7 @@ def test_datetime_time(self, sqlite_buildin): # first, use the fallback to have the sqlite adapter put in place sqlite_conn = sqlite_buildin assert sql.to_sql(df, "test_time2", sqlite_conn, index=False) == 2 - res = sql.read_sql_query(self.text("SELECT * FROM test_time2"), sqlite_conn) + res = sql.read_sql_query("SELECT * FROM test_time2", sqlite_conn) ref = df.applymap(lambda _: _.strftime("%H:%M:%S.%f")) tm.assert_frame_equal(ref, res) # check if adapter is in place # then test if sqlalchemy is unaffected by the sqlite adapter From 24362446480d4ee898bc5aed82434d407e22e676 Mon Sep 17 00:00:00 2001 From: Patrick Hoefler Date: Fri, 6 Jan 2023 12:04:46 +0100 Subject: [PATCH 13/15] Pin alchemy --- ci/deps/actions-310.yaml | 2 +- ci/deps/actions-38-downstream_compat.yaml | 2 +- ci/deps/actions-38.yaml | 2 +- ci/deps/actions-39.yaml | 2 +- ci/deps/circle-38-arm64.yaml | 2 +- environment.yml | 2 +- requirements-dev.txt | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ci/deps/actions-310.yaml b/ci/deps/actions-310.yaml index d787571d9d112..35448752119ad 100644 --- a/ci/deps/actions-310.yaml +++ b/ci/deps/actions-310.yaml @@ -48,7 +48,7 @@ dependencies: - pyxlsb - s3fs>=2021.08.0 - scipy - - sqlalchemy + - sqlalchemy=1.4.46 - tabulate - tzdata>=2022a - xarray diff --git a/ci/deps/actions-38-downstream_compat.yaml b/ci/deps/actions-38-downstream_compat.yaml index 95ec98d72ebcc..a07113d7e1b9b 100644 --- a/ci/deps/actions-38-downstream_compat.yaml +++ b/ci/deps/actions-38-downstream_compat.yaml @@ -48,7 +48,7 @@ dependencies: - pyxlsb - s3fs>=2021.08.0 - scipy - - sqlalchemy + - sqlalchemy=1.4.46 - tabulate - xarray - xlrd diff --git a/ci/deps/actions-38.yaml b/ci/deps/actions-38.yaml index f7de8bbee7d8a..1a736c0c58586 100644 --- a/ci/deps/actions-38.yaml +++ b/ci/deps/actions-38.yaml @@ -48,7 +48,7 @@ dependencies: - pyxlsb - s3fs>=2021.08.0 - scipy - - sqlalchemy + - sqlalchemy=1.4.46 - tabulate - xarray - xlrd diff --git a/ci/deps/actions-39.yaml b/ci/deps/actions-39.yaml index 821ec9c5d4234..85d73743e57e3 100644 --- a/ci/deps/actions-39.yaml +++ b/ci/deps/actions-39.yaml @@ -48,7 +48,7 @@ dependencies: - pyxlsb - s3fs>=2021.08.0 - scipy - - sqlalchemy + - sqlalchemy=1.4.46 - tabulate - tzdata>=2022a - xarray diff --git a/ci/deps/circle-38-arm64.yaml b/ci/deps/circle-38-arm64.yaml index c94ce79ea2ff8..5cce839ca1e66 100644 --- a/ci/deps/circle-38-arm64.yaml +++ b/ci/deps/circle-38-arm64.yaml @@ -49,7 +49,7 @@ dependencies: - pyxlsb - s3fs>=2021.08.0 - scipy - - sqlalchemy + - sqlalchemy=1.4.46 - tabulate - xarray - xlrd diff --git a/environment.yml b/environment.yml index b6b8f7d6af1ba..f3aadf20265cb 100644 --- a/environment.yml +++ b/environment.yml @@ -51,7 +51,7 @@ dependencies: - pyxlsb - s3fs>=2021.08.0 - scipy - - sqlalchemy + - sqlalchemy>=1.4.46 - tabulate - tzdata>=2022a - xarray diff --git a/requirements-dev.txt b/requirements-dev.txt index 4f2a80d932fd0..869a4e5472d8b 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -40,7 +40,7 @@ python-snappy pyxlsb s3fs>=2021.08.0 scipy -sqlalchemy +sqlalchemy>=1.4.46 tabulate tzdata>=2022.1 xarray From 1903bf6a8f91f35262d9404603d376e96723d421 Mon Sep 17 00:00:00 2001 From: Patrick Hoefler Date: Fri, 6 Jan 2023 12:04:53 +0100 Subject: [PATCH 14/15] Change tests --- pandas/tests/io/test_sql.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index ffac2b1d014ec..327fc20789601 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -1682,12 +1682,12 @@ def test_create_table(self): from sqlalchemy import inspect temp_conn = self.connect() - temp_conn.begin() - temp_frame = DataFrame( - {"one": [1.0, 2.0, 3.0, 4.0], "two": [4.0, 3.0, 2.0, 1.0]} - ) - pandasSQL = sql.SQLDatabase(temp_conn) - assert pandasSQL.to_sql(temp_frame, "temp_frame") == 4 + with temp_conn.begin(): + temp_frame = DataFrame( + {"one": [1.0, 2.0, 3.0, 4.0], "two": [4.0, 3.0, 2.0, 1.0]} + ) + pandasSQL = sql.SQLDatabase(temp_conn) + assert pandasSQL.to_sql(temp_frame, "temp_frame") == 4 insp = inspect(temp_conn) assert insp.has_table("temp_frame") @@ -1696,12 +1696,12 @@ def test_drop_table(self): from sqlalchemy import inspect temp_conn = self.connect() - temp_conn.begin() - temp_frame = DataFrame( - {"one": [1.0, 2.0, 3.0, 4.0], "two": [4.0, 3.0, 2.0, 1.0]} - ) - pandasSQL = sql.SQLDatabase(temp_conn) - assert pandasSQL.to_sql(temp_frame, "temp_frame") == 4 + with temp_conn.begin(): + temp_frame = DataFrame( + {"one": [1.0, 2.0, 3.0, 4.0], "two": [4.0, 3.0, 2.0, 1.0]} + ) + pandasSQL = sql.SQLDatabase(temp_conn) + assert pandasSQL.to_sql(temp_frame, "temp_frame") == 4 insp = inspect(temp_conn) assert insp.has_table("temp_frame") From 5088289855ecfe148052938f604c95b58ebcc147 Mon Sep 17 00:00:00 2001 From: Patrick Hoefler Date: Fri, 6 Jan 2023 13:42:15 +0100 Subject: [PATCH 15/15] Fix --- pandas/tests/io/test_sql.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 327fc20789601..5f1b56643c158 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -1669,9 +1669,11 @@ def setup_driver(cls): def setup_engine(cls): raise NotImplementedError() + @pytest.mark.skip(reason="sorted on strings and integers") def test_read_sql_parameter(self): self._read_sql_iris_parameter() + @pytest.mark.skip(reason="sorted on strings and integers") def test_read_sql_named_parameter(self): self._read_sql_iris_named_parameter() @@ -2439,10 +2441,6 @@ def setup_driver(cls): # sqlite3 is built-in cls.driver = None - @pytest.mark.skip(reason="sorted on strings and integers") - def test_read_sql_parameter(self): - self._read_sql_iris_parameter() - def test_default_type_conversion(self): df = sql.read_sql_table("types", self.conn)