From ebff383e950a55873c00c490794205b9227747ac Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Wed, 14 Jun 2023 15:19:32 -0700 Subject: [PATCH 1/3] TST: Use more pytest fixtures --- pandas/tests/interchange/conftest.py | 12 ---- pandas/tests/interchange/test_impl.py | 70 +++++++++---------- .../interchange/test_spec_conformance.py | 11 +++ pandas/tests/io/excel/test_xlrd.py | 4 +- pandas/tests/io/formats/style/test_html.py | 12 ++-- pandas/tests/io/formats/test_format.py | 7 -- .../io/parser/usecols/test_parse_dates.py | 9 --- .../tests/io/parser/usecols/test_strings.py | 15 ++-- pandas/tests/io/pytables/test_round_trip.py | 7 +- pandas/tests/io/pytables/test_store.py | 2 - pandas/tests/io/test_sql.py | 55 ++++++++------- pandas/tests/reductions/test_reductions.py | 5 +- 12 files changed, 92 insertions(+), 117 deletions(-) delete mode 100644 pandas/tests/interchange/conftest.py diff --git a/pandas/tests/interchange/conftest.py b/pandas/tests/interchange/conftest.py deleted file mode 100644 index f552ba442a916..0000000000000 --- a/pandas/tests/interchange/conftest.py +++ /dev/null @@ -1,12 +0,0 @@ -import pytest - -import pandas as pd - - -@pytest.fixture -def df_from_dict(): - def maker(dct, is_categorical=False): - df = pd.DataFrame(dct) - return df.astype("category") if is_categorical else df - - return maker diff --git a/pandas/tests/interchange/test_impl.py b/pandas/tests/interchange/test_impl.py index 49873768ca952..5fce4f162d71f 100644 --- a/pandas/tests/interchange/test_impl.py +++ b/pandas/tests/interchange/test_impl.py @@ -16,47 +16,31 @@ ) from pandas.core.interchange.from_dataframe import from_dataframe -test_data_categorical = { - "ordered": pd.Categorical(list("testdata") * 30, ordered=True), - "unordered": pd.Categorical(list("testdata") * 30, ordered=False), -} -NCOLS, NROWS = 100, 200 - - -def _make_data(make_one): +@pytest.fixture +def data_categorical(): return { - f"col{int((i - NCOLS / 2) % NCOLS + 1)}": [make_one() for _ in range(NROWS)] - for i in range(NCOLS) + "ordered": pd.Categorical(list("testdata") * 30, ordered=True), + "unordered": pd.Categorical(list("testdata") * 30, ordered=False), } -int_data = _make_data(lambda: random.randint(-100, 100)) -uint_data = _make_data(lambda: random.randint(1, 100)) -bool_data = _make_data(lambda: random.choice([True, False])) -float_data = _make_data(lambda: random.random()) -datetime_data = _make_data( - lambda: datetime( - year=random.randint(1900, 2100), - month=random.randint(1, 12), - day=random.randint(1, 20), - ) -) - -string_data = { - "separator data": [ - "abC|DeF,Hik", - "234,3245.67", - "gSaf,qWer|Gre", - "asd3,4sad|", - np.NaN, - ] -} +@pytest.fixture +def string_data(): + return { + "separator data": [ + "abC|DeF,Hik", + "234,3245.67", + "gSaf,qWer|Gre", + "asd3,4sad|", + np.NaN, + ] + } @pytest.mark.parametrize("data", [("ordered", True), ("unordered", False)]) -def test_categorical_dtype(data): - df = pd.DataFrame({"A": (test_data_categorical[data[0]])}) +def test_categorical_dtype(data, data_categorical): + df = pd.DataFrame({"A": (data_categorical[data[0]])}) col = df.__dataframe__().get_column_by_name("A") assert col.dtype[0] == DtypeKind.CATEGORICAL @@ -143,9 +127,25 @@ def test_bitmasks_pyarrow(offset, length, expected_values): @pytest.mark.parametrize( - "data", [int_data, uint_data, float_data, bool_data, datetime_data] + "data", + [ + lambda: random.randint(-100, 100), + lambda: random.randint(1, 100), + lambda: random.random(), + lambda: random.choice([True, False]), + lambda: datetime( + year=random.randint(1900, 2100), + month=random.randint(1, 12), + day=random.randint(1, 20), + ), + ], ) def test_dataframe(data): + NCOLS, NROWS = 10, 20 + data = { + f"col{int((i - NCOLS / 2) % NCOLS + 1)}": [data() for _ in range(NROWS)] + for i in range(NCOLS) + } df = pd.DataFrame(data) df2 = df.__dataframe__() @@ -227,7 +227,7 @@ def test_mixed_missing(): assert df2.get_column_by_name(col_name).null_count == 2 -def test_string(): +def test_string(string_data): test_str_data = string_data["separator data"] + [""] df = pd.DataFrame({"A": test_str_data}) col = df.__dataframe__().get_column_by_name("A") diff --git a/pandas/tests/interchange/test_spec_conformance.py b/pandas/tests/interchange/test_spec_conformance.py index 965938b111e86..7c02379c11853 100644 --- a/pandas/tests/interchange/test_spec_conformance.py +++ b/pandas/tests/interchange/test_spec_conformance.py @@ -7,6 +7,17 @@ import pytest +import pandas as pd + + +@pytest.fixture +def df_from_dict(): + def maker(dct, is_categorical=False): + df = pd.DataFrame(dct) + return df.astype("category") if is_categorical else df + + return maker + @pytest.mark.parametrize( "test_data", diff --git a/pandas/tests/io/excel/test_xlrd.py b/pandas/tests/io/excel/test_xlrd.py index 1f8fb4b801356..509029861715e 100644 --- a/pandas/tests/io/excel/test_xlrd.py +++ b/pandas/tests/io/excel/test_xlrd.py @@ -10,10 +10,8 @@ xlrd = pytest.importorskip("xlrd") -exts = [".xls"] - -@pytest.fixture(params=exts) +@pytest.fixture(params=[".xls"]) def read_ext_xlrd(request): """ Valid extensions for reading Excel files with xlrd. diff --git a/pandas/tests/io/formats/style/test_html.py b/pandas/tests/io/formats/style/test_html.py index 53062c52a29db..1e345eb82ed3c 100644 --- a/pandas/tests/io/formats/style/test_html.py +++ b/pandas/tests/io/formats/style/test_html.py @@ -15,8 +15,12 @@ jinja2 = pytest.importorskip("jinja2") from pandas.io.formats.style import Styler -loader = jinja2.PackageLoader("pandas", "io/formats/templates") -env = jinja2.Environment(loader=loader, trim_blocks=True) + +@pytest.fixture +def env(): + loader = jinja2.PackageLoader("pandas", "io/formats/templates") + env = jinja2.Environment(loader=loader, trim_blocks=True) + return env @pytest.fixture @@ -31,12 +35,12 @@ def styler_mi(): @pytest.fixture -def tpl_style(): +def tpl_style(env): return env.get_template("html_style.tpl") @pytest.fixture -def tpl_table(): +def tpl_table(env): return env.get_template("html_table.tpl") diff --git a/pandas/tests/io/formats/test_format.py b/pandas/tests/io/formats/test_format.py index 6acef0f564ef4..7436644b8636a 100644 --- a/pandas/tests/io/formats/test_format.py +++ b/pandas/tests/io/formats/test_format.py @@ -24,11 +24,6 @@ from pandas._config import config -from pandas.compat import ( - IS64, - is_platform_windows, -) - import pandas as pd from pandas import ( DataFrame, @@ -48,8 +43,6 @@ from pandas.io.formats import printing import pandas.io.formats.format as fmt -use_32bit_repr = is_platform_windows() or not IS64 - def get_local_am_pm(): """Return the AM and PM strings returned by strftime in current locale.""" diff --git a/pandas/tests/io/parser/usecols/test_parse_dates.py b/pandas/tests/io/parser/usecols/test_parse_dates.py index 32231cbbdda64..069ac2a69d224 100644 --- a/pandas/tests/io/parser/usecols/test_parse_dates.py +++ b/pandas/tests/io/parser/usecols/test_parse_dates.py @@ -13,15 +13,6 @@ ) import pandas._testing as tm -_msg_validate_usecols_arg = ( - "'usecols' must either be list-like " - "of all strings, all unicode, all " - "integers or a callable." -) -_msg_validate_usecols_names = ( - "Usecols do not match columns, columns expected but not found: {0}" -) - # TODO(1.4): Change these to xfails whenever parse_dates support(which was # intentionally disable to keep small PR sizes) is added back pytestmark = pytest.mark.usefixtures("pyarrow_skip") diff --git a/pandas/tests/io/parser/usecols/test_strings.py b/pandas/tests/io/parser/usecols/test_strings.py index 8cecf1fc981ee..22f19ec518e4a 100644 --- a/pandas/tests/io/parser/usecols/test_strings.py +++ b/pandas/tests/io/parser/usecols/test_strings.py @@ -9,15 +9,6 @@ from pandas import DataFrame import pandas._testing as tm -_msg_validate_usecols_arg = ( - "'usecols' must either be list-like " - "of all strings, all unicode, all " - "integers or a callable." -) -_msg_validate_usecols_names = ( - "Usecols do not match columns, columns expected but not found: {0}" -) - def test_usecols_with_unicode_strings(all_parsers): # see gh-13219 @@ -70,7 +61,11 @@ def test_usecols_with_mixed_encoding_strings(all_parsers, usecols): 2.613230982,2,False,b 3.568935038,7,False,a""" parser = all_parsers - + _msg_validate_usecols_arg = ( + "'usecols' must either be list-like " + "of all strings, all unicode, all " + "integers or a callable." + ) with pytest.raises(ValueError, match=_msg_validate_usecols_arg): parser.read_csv(StringIO(data), usecols=usecols) diff --git a/pandas/tests/io/pytables/test_round_trip.py b/pandas/tests/io/pytables/test_round_trip.py index 2461f937c9eff..42f020a8f3708 100644 --- a/pandas/tests/io/pytables/test_round_trip.py +++ b/pandas/tests/io/pytables/test_round_trip.py @@ -26,9 +26,6 @@ ) from pandas.util import _test_decorators as td -_default_compressor = "blosc" - - pytestmark = pytest.mark.single_cpu @@ -479,7 +476,7 @@ def _make_one(): def _check_roundtrip(obj, comparator, path, compression=False, **kwargs): options = {} if compression: - options["complib"] = _default_compressor + options["complib"] = "blosc" with ensure_clean_store(path, "w", **options) as store: store["obj"] = obj @@ -490,7 +487,7 @@ def _check_roundtrip(obj, comparator, path, compression=False, **kwargs): def _check_roundtrip_table(obj, comparator, path, compression=False): options = {} if compression: - options["complib"] = _default_compressor + options["complib"] = "blosc" with ensure_clean_store(path, "w", **options) as store: store.put("obj", obj, format="table") diff --git a/pandas/tests/io/pytables/test_store.py b/pandas/tests/io/pytables/test_store.py index 2d87b719af36b..82330e1d63c9a 100644 --- a/pandas/tests/io/pytables/test_store.py +++ b/pandas/tests/io/pytables/test_store.py @@ -30,8 +30,6 @@ safe_close, ) -_default_compressor = "blosc" - from pandas.io.pytables import ( HDFStore, read_hdf, diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 7a3f7521d4a17..2d78070bb0030 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -75,31 +75,34 @@ except ImportError: SQLALCHEMY_INSTALLED = False -SQL_STRINGS = { - "read_parameters": { - "sqlite": "SELECT * FROM iris WHERE Name=? AND SepalLength=?", - "mysql": "SELECT * FROM iris WHERE `Name`=%s AND `SepalLength`=%s", - "postgresql": 'SELECT * FROM iris WHERE "Name"=%s AND "SepalLength"=%s', - }, - "read_named_parameters": { - "sqlite": """ + +@pytest.fixture +def sql_strings(): + return { + "read_parameters": { + "sqlite": "SELECT * FROM iris WHERE Name=? AND SepalLength=?", + "mysql": "SELECT * FROM iris WHERE `Name`=%s AND `SepalLength`=%s", + "postgresql": 'SELECT * FROM iris WHERE "Name"=%s AND "SepalLength"=%s', + }, + "read_named_parameters": { + "sqlite": """ SELECT * FROM iris WHERE Name=:name AND SepalLength=:length """, - "mysql": """ + "mysql": """ SELECT * FROM iris WHERE `Name`=%(name)s AND `SepalLength`=%(length)s """, - "postgresql": """ + "postgresql": """ SELECT * FROM iris WHERE "Name"=%(name)s AND "SepalLength"=%(length)s """, - }, - "read_no_parameters_with_percent": { - "sqlite": "SELECT * FROM iris WHERE Name LIKE '%'", - "mysql": "SELECT * FROM iris WHERE `Name` LIKE '%'", - "postgresql": "SELECT * FROM iris WHERE \"Name\" LIKE '%'", - }, -} + }, + "read_no_parameters_with_percent": { + "sqlite": "SELECT * FROM iris WHERE Name LIKE '%'", + "mysql": "SELECT * FROM iris WHERE `Name` LIKE '%'", + "postgresql": "SELECT * FROM iris WHERE \"Name\" LIKE '%'", + }, + } def iris_table_metadata(dialect: str): @@ -669,12 +672,12 @@ def test_read_iris_query_expression_with_parameter(conn, request): @pytest.mark.db @pytest.mark.parametrize("conn", all_connectable_iris) -def test_read_iris_query_string_with_parameter(conn, request): - for db, query in SQL_STRINGS["read_parameters"].items(): +def test_read_iris_query_string_with_parameter(conn, request, sql_strings): + for db, query in sql_strings["read_parameters"].items(): if db in conn: break else: - raise KeyError(f"No part of {conn} found in SQL_STRINGS['read_parameters']") + raise KeyError(f"No part of {conn} found in sql_strings['read_parameters']") conn = request.getfixturevalue(conn) iris_frame = read_sql_query(query, conn, params=("Iris-setosa", 5.1)) check_iris_frame(iris_frame) @@ -933,20 +936,20 @@ def load_types_data(self, types_data): else: create_and_load_types(self.conn, types_data, self.flavor) - def _read_sql_iris_parameter(self): - query = SQL_STRINGS["read_parameters"][self.flavor] + def _read_sql_iris_parameter(self, sql_strings): + query = sql_strings["read_parameters"][self.flavor] params = ("Iris-setosa", 5.1) iris_frame = self.pandasSQL.read_query(query, params=params) check_iris_frame(iris_frame) - def _read_sql_iris_named_parameter(self): - query = SQL_STRINGS["read_named_parameters"][self.flavor] + def _read_sql_iris_named_parameter(self, sql_strings): + query = sql_strings["read_named_parameters"][self.flavor] params = {"name": "Iris-setosa", "length": 5.1} iris_frame = self.pandasSQL.read_query(query, params=params) check_iris_frame(iris_frame) - def _read_sql_iris_no_parameter_with_percent(self): - query = SQL_STRINGS["read_no_parameters_with_percent"][self.flavor] + def _read_sql_iris_no_parameter_with_percent(self, sql_strings): + query = sql_strings["read_no_parameters_with_percent"][self.flavor] iris_frame = self.pandasSQL.read_query(query, params=None) check_iris_frame(iris_frame) diff --git a/pandas/tests/reductions/test_reductions.py b/pandas/tests/reductions/test_reductions.py index 4ea3c75cb684a..83b9a83c0a6a2 100644 --- a/pandas/tests/reductions/test_reductions.py +++ b/pandas/tests/reductions/test_reductions.py @@ -47,12 +47,9 @@ def get_objs(): return objs -objs = get_objs() - - class TestReductions: @pytest.mark.parametrize("opname", ["max", "min"]) - @pytest.mark.parametrize("obj", objs) + @pytest.mark.parametrize("obj", get_objs()) def test_ops(self, opname, obj): result = getattr(obj, opname)() if not isinstance(obj, PeriodIndex): From d752dcf24c3165438974f333e273645874763b9a Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Wed, 14 Jun 2023 17:08:55 -0700 Subject: [PATCH 2/3] Fix sql fixture --- pandas/tests/io/test_sql.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 2d78070bb0030..e44aa41d7ac5d 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -1835,11 +1835,11 @@ def setup_driver(cls): def setup_engine(cls): raise NotImplementedError() - def test_read_sql_parameter(self): - self._read_sql_iris_parameter() + def test_read_sql_parameter(self, sql_strings): + self._read_sql_iris_parameter(sql_strings) - def test_read_sql_named_parameter(self): - self._read_sql_iris_named_parameter() + def test_read_sql_named_parameter(self, sql_strings): + self._read_sql_iris_named_parameter(sql_strings) def test_to_sql_empty(self, test_frame1): self._to_sql_empty(test_frame1) From 818637e82b3dd619c1c1d0c9eae46103f5aaad1f Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Wed, 14 Jun 2023 18:42:03 -0700 Subject: [PATCH 3/3] More test fixtures --- pandas/tests/io/test_sql.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index e44aa41d7ac5d..a3004b0626961 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -2951,11 +2951,11 @@ def setup_method(self, iris_path, types_data): self.load_types_data(types_data) self.pandasSQL = sql.SQLiteDatabase(self.conn) - def test_read_sql_parameter(self): - self._read_sql_iris_parameter() + def test_read_sql_parameter(self, sql_strings): + self._read_sql_iris_parameter(sql_strings) - def test_read_sql_named_parameter(self): - self._read_sql_iris_named_parameter() + def test_read_sql_named_parameter(self, sql_strings): + self._read_sql_iris_named_parameter(sql_strings) def test_to_sql_empty(self, test_frame1): self._to_sql_empty(test_frame1)