diff --git a/ci/deps/azure-37-minimum_versions.yaml b/ci/deps/azure-37-minimum_versions.yaml index afd5b07cc6654..f184ea87c89fe 100644 --- a/ci/deps/azure-37-minimum_versions.yaml +++ b/ci/deps/azure-37-minimum_versions.yaml @@ -20,7 +20,7 @@ dependencies: - numexpr=2.6.8 - numpy=1.16.5 - openpyxl=2.6.0 - - pytables=3.4.4 + - pytables=3.5.1 - python-dateutil=2.7.3 - pytz=2017.3 - pyarrow=0.15 diff --git a/ci/deps/travis-37-locale.yaml b/ci/deps/travis-37-locale.yaml index cd6341e80be24..ddaf0bea097c7 100644 --- a/ci/deps/travis-37-locale.yaml +++ b/ci/deps/travis-37-locale.yaml @@ -13,7 +13,7 @@ dependencies: # pandas dependencies - beautifulsoup4 - - blosc=1.14.3 + - blosc=1.15.0 - python-blosc - fastparquet=0.3.2 - html5lib @@ -30,7 +30,7 @@ dependencies: - pyarrow>=0.17 - psycopg2=2.7 - pymysql=0.7.11 - - pytables + - pytables>=3.5.1 - python-dateutil - pytz - scipy diff --git a/doc/source/getting_started/install.rst b/doc/source/getting_started/install.rst index 78bd76bbd230f..a6341451b1b80 100644 --- a/doc/source/getting_started/install.rst +++ b/doc/source/getting_started/install.rst @@ -266,7 +266,7 @@ PyTables 3.4.4 HDF5-based reading / writing SQLAlchemy 1.2.8 SQL support for databases other than sqlite SciPy 1.12.0 Miscellaneous statistical functions xlsxwriter 1.0.2 Excel writing -blosc 1.14.3 Compression for HDF5 +blosc 1.15.0 Compression for HDF5 fsspec 0.7.4 Handling files aside from local and HTTP fastparquet 0.3.2 Parquet reading / writing gcsfs 0.6.0 Google Cloud Storage access @@ -280,7 +280,7 @@ psycopg2 2.7 PostgreSQL engine for sqlalchemy pyarrow 0.15.0 Parquet, ORC, and feather reading / writing pymysql 0.7.11 MySQL engine for sqlalchemy pyreadstat SPSS files (.sav) reading -pytables 3.4.4 HDF5 reading / writing +pytables 3.5.1 HDF5 reading / writing pyxlsb 1.0.6 Reading for xlsb files qtpy Clipboard I/O s3fs 0.4.0 Amazon S3 access diff --git a/doc/source/whatsnew/v1.2.0.rst b/doc/source/whatsnew/v1.2.0.rst index 88ad1dde5c9b0..381e2c5b12218 100644 --- a/doc/source/whatsnew/v1.2.0.rst +++ b/doc/source/whatsnew/v1.2.0.rst @@ -181,7 +181,7 @@ Optional libraries below the lowest tested version may still work, but are not c +-----------------+-----------------+---------+ | pymysql | 0.7.11 | X | +-----------------+-----------------+---------+ -| pytables | 3.4.4 | X | +| pytables | 3.5.1 | X | +-----------------+-----------------+---------+ | s3fs | 0.4.0 | | +-----------------+-----------------+---------+ @@ -328,6 +328,7 @@ I/O - Bug in :func:`LongTableBuilder.middle_separator` was duplicating LaTeX longtable entries in the List of Tables of a LaTeX document (:issue:`34360`) - Bug in :meth:`read_csv` with ``engine='python'`` truncating data if multiple items present in first row and first element started with BOM (:issue:`36343`) - Removed ``private_key`` and ``verbose`` from :func:`read_gbq` as they are no longer supported in ``pandas-gbq`` (:issue:`34654`, :issue:`30200`) +- Bumped minimum pytables version to 3.5.1 to avoid a ``ValueError`` in :meth:`read_hdf` (:issue:`24839`) Plotting ^^^^^^^^ diff --git a/environment.yml b/environment.yml index f97f8e2457585..6e9b417beb0af 100644 --- a/environment.yml +++ b/environment.yml @@ -100,7 +100,7 @@ dependencies: - python-snappy # required by pyarrow - pyqt>=5.9.2 # pandas.read_clipboard - - pytables>=3.4.4 # pandas.read_hdf, DataFrame.to_hdf + - pytables>=3.5.1 # pandas.read_hdf, DataFrame.to_hdf - s3fs>=0.4.0 # file IO when using 's3://...' path - fsspec>=0.7.4 # for generic remote file operations - gcsfs>=0.6.0 # file IO when using 'gcs://...' path diff --git a/pandas/tests/io/pytables/test_complex.py b/pandas/tests/io/pytables/test_complex.py index 543940e674dba..3a7aff3b551c2 100644 --- a/pandas/tests/io/pytables/test_complex.py +++ b/pandas/tests/io/pytables/test_complex.py @@ -3,8 +3,6 @@ import numpy as np import pytest -import pandas.util._test_decorators as td - import pandas as pd from pandas import DataFrame, Series import pandas._testing as tm @@ -12,8 +10,6 @@ from pandas.io.pytables import read_hdf -# GH10447 - def test_complex_fixed(setup_path): df = DataFrame( @@ -62,7 +58,6 @@ def test_complex_table(setup_path): tm.assert_frame_equal(df, reread) -@td.xfail_non_writeable def test_complex_mixed_fixed(setup_path): complex64 = np.array( [1.0 + 1.0j, 1.0 + 1.0j, 1.0 + 1.0j, 1.0 + 1.0j], dtype=np.complex64 diff --git a/pandas/tests/io/pytables/test_store.py b/pandas/tests/io/pytables/test_store.py index 0942c79837e7c..11ef698241144 100644 --- a/pandas/tests/io/pytables/test_store.py +++ b/pandas/tests/io/pytables/test_store.py @@ -909,7 +909,6 @@ def test_put_integer(self, setup_path): df = DataFrame(np.random.randn(50, 100)) self._check_roundtrip(df, tm.assert_frame_equal, setup_path) - @td.xfail_non_writeable def test_put_mixed_type(self, setup_path): df = tm.makeTimeDataFrame() df["obj1"] = "foo" @@ -1518,9 +1517,7 @@ def test_to_hdf_with_min_itemsize(self, setup_path): pd.read_hdf(path, "ss4"), pd.concat([df["B"], df2["B"]]) ) - @pytest.mark.parametrize( - "format", [pytest.param("fixed", marks=td.xfail_non_writeable), "table"] - ) + @pytest.mark.parametrize("format", ["fixed", "table"]) def test_to_hdf_errors(self, format, setup_path): data = ["\ud800foo"] @@ -1956,7 +1953,6 @@ def test_pass_spec_to_storer(self, setup_path): with pytest.raises(TypeError): store.select("df", where=[("columns=A")]) - @td.xfail_non_writeable def test_append_misc(self, setup_path): with ensure_clean_store(setup_path) as store: @@ -2164,14 +2160,6 @@ def test_unimplemented_dtypes_table_columns(self, setup_path): with pytest.raises(TypeError): store.append("df_unimplemented", df) - @td.xfail_non_writeable - @pytest.mark.skipif( - LooseVersion(np.__version__) == LooseVersion("1.15.0"), - reason=( - "Skipping pytables test when numpy version is " - "exactly equal to 1.15.0: gh-22098" - ), - ) def test_calendar_roundtrip_issue(self, setup_path): # 8591 @@ -2399,7 +2387,6 @@ def test_float_index(self, setup_path): s = Series(np.random.randn(10), index=index) self._check_roundtrip(s, tm.assert_series_equal, path=setup_path) - @td.xfail_non_writeable def test_tuple_index(self, setup_path): # GH #492 @@ -2412,7 +2399,6 @@ def test_tuple_index(self, setup_path): simplefilter("ignore", pd.errors.PerformanceWarning) self._check_roundtrip(DF, tm.assert_frame_equal, path=setup_path) - @td.xfail_non_writeable @pytest.mark.filterwarnings("ignore::pandas.errors.PerformanceWarning") def test_index_types(self, setup_path): @@ -2474,7 +2460,6 @@ def test_timeseries_preepoch(self, setup_path): except OverflowError: pytest.skip("known failer on some windows platforms") - @td.xfail_non_writeable @pytest.mark.parametrize( "compression", [False, pytest.param(True, marks=td.skip_if_windows_python_3)] ) @@ -2508,7 +2493,6 @@ def test_frame(self, compression, setup_path): # empty self._check_roundtrip(df[:0], tm.assert_frame_equal, path=setup_path) - @td.xfail_non_writeable def test_empty_series_frame(self, setup_path): s0 = Series(dtype=object) s1 = Series(name="myseries", dtype=object) @@ -2522,7 +2506,6 @@ def test_empty_series_frame(self, setup_path): self._check_roundtrip(df1, tm.assert_frame_equal, path=setup_path) self._check_roundtrip(df2, tm.assert_frame_equal, path=setup_path) - @td.xfail_non_writeable @pytest.mark.parametrize( "dtype", [np.int64, np.float64, object, "m8[ns]", "M8[ns]"] ) @@ -2608,7 +2591,6 @@ def test_store_series_name(self, setup_path): recons = store["series"] tm.assert_series_equal(recons, series) - @td.xfail_non_writeable @pytest.mark.parametrize( "compression", [False, pytest.param(True, marks=td.skip_if_windows_python_3)] ) @@ -4176,7 +4158,6 @@ def test_pytables_native2_read(self, datapath, setup_path): d1 = store["detector"] assert isinstance(d1, DataFrame) - @td.xfail_non_writeable def test_legacy_table_fixed_format_read_py2(self, datapath, setup_path): # GH 24510 # legacy table with fixed format written in Python 2 @@ -4350,7 +4331,6 @@ def test_unicode_longer_encoded(self, setup_path): result = store.get("df") tm.assert_frame_equal(result, df) - @td.xfail_non_writeable def test_store_datetime_mixed(self, setup_path): df = DataFrame({"a": [1, 2, 3], "b": [1.0, 2.0, 3.0], "c": ["a", "b", "c"]}) diff --git a/pandas/util/_test_decorators.py b/pandas/util/_test_decorators.py index 0e8f6b933cd97..e3b779678c68b 100644 --- a/pandas/util/_test_decorators.py +++ b/pandas/util/_test_decorators.py @@ -75,21 +75,6 @@ def safe_import(mod_name: str, min_version: Optional[str] = None): return False -# TODO: -# remove when gh-24839 is fixed. -# this affects numpy 1.16 and pytables 3.4.4 -tables = safe_import("tables") -xfail_non_writeable = pytest.mark.xfail( - tables - and LooseVersion(np.__version__) >= LooseVersion("1.16") - and LooseVersion(tables.__version__) < LooseVersion("3.5.1"), - reason=( - "gh-25511, gh-24839. pytables needs a " - "release beyond 3.4.4 to support numpy 1.16.x" - ), -) - - def _skip_if_no_mpl(): mod = safe_import("matplotlib") if mod: diff --git a/requirements-dev.txt b/requirements-dev.txt index 5a1c6a80334ed..8f3dd20f309aa 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -67,7 +67,7 @@ fastparquet>=0.3.2 pyarrow>=0.15.0 python-snappy pyqt5>=5.9.2 -tables>=3.4.4 +tables>=3.5.1 s3fs>=0.4.0 fsspec>=0.7.4 gcsfs>=0.6.0