From c391e3b8cb6e3fe85e2fdb081d2bb4454470be3c Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Tue, 20 Jun 2023 15:35:28 -0700 Subject: [PATCH 1/7] TST: Make test_complibs deterministic --- pandas/tests/io/pytables/test_file_handling.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pandas/tests/io/pytables/test_file_handling.py b/pandas/tests/io/pytables/test_file_handling.py index 49190daa37442..19255166750c1 100644 --- a/pandas/tests/io/pytables/test_file_handling.py +++ b/pandas/tests/io/pytables/test_file_handling.py @@ -224,7 +224,9 @@ def test_complibs_default_settings_override(tmp_path, setup_path): def test_complibs(tmp_path, setup_path): # GH14478 - df = tm.makeDataFrame() + df = DataFrame( + np.ones((30, 4)), columns=list("ABCD"), index=np.arange(30).astype(np.str_) + ) # Building list of all complibs and complevels tuples all_complibs = tables.filters.all_complibs From 4818a5849a047ecdd14bdd10ea502f0c18c3da99 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Tue, 20 Jun 2023 17:35:04 -0700 Subject: [PATCH 2/7] Make sure files are unique? --- .../tests/io/pytables/test_file_handling.py | 42 +++++++++---------- 1 file changed, 19 insertions(+), 23 deletions(-) diff --git a/pandas/tests/io/pytables/test_file_handling.py b/pandas/tests/io/pytables/test_file_handling.py index 19255166750c1..7de5c383ccd96 100644 --- a/pandas/tests/io/pytables/test_file_handling.py +++ b/pandas/tests/io/pytables/test_file_handling.py @@ -222,41 +222,37 @@ def test_complibs_default_settings_override(tmp_path, setup_path): assert node.filters.complib == "blosc" -def test_complibs(tmp_path, setup_path): +@pytest.mark.parametrize("lvl", range(10)) +@pytest.mark.parametrize("lib", tables.filters.all_complibs) +def test_complibs(tmp_path, lvl, lib): # GH14478 df = DataFrame( np.ones((30, 4)), columns=list("ABCD"), index=np.arange(30).astype(np.str_) ) - # Building list of all complibs and complevels tuples - all_complibs = tables.filters.all_complibs # Remove lzo if its not available on this platform if not tables.which_lib_version("lzo"): - all_complibs.remove("lzo") + pytest.skip("lzo not available") # Remove bzip2 if its not available on this platform if not tables.which_lib_version("bzip2"): - all_complibs.remove("bzip2") + pytest.skip("bzip2 not available") - all_levels = range(0, 10) - all_tests = [(lib, lvl) for lib in all_complibs for lvl in all_levels] + tmpfile = tmp_path / f"{lvl}_{lib}.h5" + gname = "foo" - for lib, lvl in all_tests: - tmpfile = tmp_path / setup_path - gname = "foo" - - # Write and read file to see if data is consistent - df.to_hdf(tmpfile, gname, complib=lib, complevel=lvl) - result = read_hdf(tmpfile, gname) - tm.assert_frame_equal(result, df) + # Write and read file to see if data is consistent + df.to_hdf(tmpfile, gname, complib=lib, complevel=lvl) + result = read_hdf(tmpfile, gname) + tm.assert_frame_equal(result, df) - # Open file and check metadata for correct amount of compression - with tables.open_file(tmpfile, mode="r") as h5table: - for node in h5table.walk_nodes(where="/" + gname, classname="Leaf"): - assert node.filters.complevel == lvl - if lvl == 0: - assert node.filters.complib is None - else: - assert node.filters.complib == lib + # Open file and check metadata for correct amount of compression + with tables.open_file(tmpfile, mode="r") as h5table: + for node in h5table.walk_nodes(where="/" + gname, classname="Leaf"): + assert node.filters.complevel == lvl + if lvl == 0: + assert node.filters.complib is None + else: + assert node.filters.complib == lib @pytest.mark.skipif( From f9529a943f40c7a220813b30d50bb4c2e36d2c69 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Wed, 21 Jun 2023 09:08:19 -0700 Subject: [PATCH 3/7] Try unique key --- pandas/tests/io/pytables/test_file_handling.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/io/pytables/test_file_handling.py b/pandas/tests/io/pytables/test_file_handling.py index 7de5c383ccd96..c56c20bf7f0bd 100644 --- a/pandas/tests/io/pytables/test_file_handling.py +++ b/pandas/tests/io/pytables/test_file_handling.py @@ -238,7 +238,7 @@ def test_complibs(tmp_path, lvl, lib): pytest.skip("bzip2 not available") tmpfile = tmp_path / f"{lvl}_{lib}.h5" - gname = "foo" + gname = f"{lvl}_{lib}" # Write and read file to see if data is consistent df.to_hdf(tmpfile, gname, complib=lib, complevel=lvl) From a8d6bf8725498f29ea56f2eade3790ef6abb91b6 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Wed, 21 Jun 2023 11:08:23 -0700 Subject: [PATCH 4/7] Just xfail test --- pandas/tests/io/pytables/test_file_handling.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/pandas/tests/io/pytables/test_file_handling.py b/pandas/tests/io/pytables/test_file_handling.py index c56c20bf7f0bd..7f4c40a9ad872 100644 --- a/pandas/tests/io/pytables/test_file_handling.py +++ b/pandas/tests/io/pytables/test_file_handling.py @@ -3,7 +3,12 @@ import numpy as np import pytest -from pandas.compat import is_platform_little_endian +from pandas.compat import ( + PY311, + is_ci_environment, + is_platform_linux, + is_platform_little_endian, +) from pandas.errors import ( ClosedFileError, PossibleDataLossError, @@ -224,6 +229,12 @@ def test_complibs_default_settings_override(tmp_path, setup_path): @pytest.mark.parametrize("lvl", range(10)) @pytest.mark.parametrize("lib", tables.filters.all_complibs) +@pytest.mark.xfail( + not PY311 and is_ci_environment() and is_platform_linux(), + reason="producing invalid start bytes", + raises=UnicodeDecodeError, + strict=False, +) def test_complibs(tmp_path, lvl, lib): # GH14478 df = DataFrame( From 04ff1aa4e56fe320c1aaf2850a72f46d72a42462 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Wed, 21 Jun 2023 14:09:10 -0700 Subject: [PATCH 5/7] Check what is taking long --- ci/run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/run_tests.sh b/ci/run_tests.sh index 42caebc19e176..c3e54a4ac1f24 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -20,7 +20,7 @@ if [[ $(uname) == "Linux" && -z $DISPLAY ]]; then XVFB="xvfb-run " fi -PYTEST_CMD="MESONPY_EDITABLE_VERBOSE=1 ${XVFB}pytest -r fEs -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET" +PYTEST_CMD="MESONPY_EDITABLE_VERBOSE=1 ${XVFB}pytest -v -r fEs -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET" if [[ "$PATTERN" ]]; then PYTEST_CMD="$PYTEST_CMD -m \"$PATTERN\"" From 91318163bc3f834fb7b3b415b809df3572fcdbc5 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Wed, 21 Jun 2023 18:06:27 -0700 Subject: [PATCH 6/7] Remove -v --- ci/run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/run_tests.sh b/ci/run_tests.sh index c3e54a4ac1f24..42caebc19e176 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -20,7 +20,7 @@ if [[ $(uname) == "Linux" && -z $DISPLAY ]]; then XVFB="xvfb-run " fi -PYTEST_CMD="MESONPY_EDITABLE_VERBOSE=1 ${XVFB}pytest -v -r fEs -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET" +PYTEST_CMD="MESONPY_EDITABLE_VERBOSE=1 ${XVFB}pytest -r fEs -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET" if [[ "$PATTERN" ]]; then PYTEST_CMD="$PYTEST_CMD -m \"$PATTERN\"" From 0811c3279055a473e69fdee8def60492e19938ae Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Wed, 21 Jun 2023 20:19:07 -0700 Subject: [PATCH 7/7] Filter warning --- pandas/tests/io/pytables/test_file_handling.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pandas/tests/io/pytables/test_file_handling.py b/pandas/tests/io/pytables/test_file_handling.py index 7f4c40a9ad872..1a126ad75c01c 100644 --- a/pandas/tests/io/pytables/test_file_handling.py +++ b/pandas/tests/io/pytables/test_file_handling.py @@ -229,6 +229,7 @@ def test_complibs_default_settings_override(tmp_path, setup_path): @pytest.mark.parametrize("lvl", range(10)) @pytest.mark.parametrize("lib", tables.filters.all_complibs) +@pytest.mark.filterwarnings("ignore:object name is not a valid") @pytest.mark.xfail( not PY311 and is_ci_environment() and is_platform_linux(), reason="producing invalid start bytes",