From 10aa9adf6c58d9a860a1d62266b3e2616753b384 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Fri, 11 Jun 2021 10:59:19 +0200 Subject: [PATCH] [ArrayManager] Enable pytables IO by falling back to BlockManager --- pandas/io/pytables.py | 14 +++++++++++++- pandas/tests/io/pytables/test_append.py | 6 +++++- pandas/tests/io/pytables/test_categorical.py | 3 --- pandas/tests/io/pytables/test_compat.py | 4 ---- pandas/tests/io/pytables/test_complex.py | 5 ----- pandas/tests/io/pytables/test_errors.py | 4 +--- pandas/tests/io/pytables/test_file_handling.py | 3 +-- pandas/tests/io/pytables/test_keys.py | 4 +--- pandas/tests/io/pytables/test_put.py | 2 +- pandas/tests/io/pytables/test_pytables_missing.py | 2 -- pandas/tests/io/pytables/test_read.py | 2 +- pandas/tests/io/pytables/test_retain_attributes.py | 3 +-- pandas/tests/io/pytables/test_round_trip.py | 2 +- pandas/tests/io/pytables/test_select.py | 3 +-- pandas/tests/io/pytables/test_store.py | 5 +---- pandas/tests/io/pytables/test_subclass.py | 4 ---- pandas/tests/io/pytables/test_time_series.py | 4 +--- pandas/tests/io/pytables/test_timezones.py | 3 --- 18 files changed, 28 insertions(+), 45 deletions(-) diff --git a/pandas/io/pytables.py b/pandas/io/pytables.py index 0eae6ea5d6b7b..208b8a008ffe6 100644 --- a/pandas/io/pytables.py +++ b/pandas/io/pytables.py @@ -86,7 +86,10 @@ ) from pandas.core.construction import extract_array from pandas.core.indexes.api import ensure_index -from pandas.core.internals import BlockManager +from pandas.core.internals import ( + ArrayManager, + BlockManager, +) from pandas.io.common import stringify_path from pandas.io.formats.printing import ( @@ -3206,6 +3209,11 @@ def read( def write(self, obj, **kwargs): super().write(obj, **kwargs) + + # TODO(ArrayManager) HDFStore relies on accessing the blocks + if isinstance(obj._mgr, ArrayManager): + obj = obj._as_manager("block") + data = obj._mgr if not data.is_consolidated(): data = data.consolidate() @@ -4015,6 +4023,10 @@ def _get_blocks_and_items( ): # Helper to clarify non-state-altering parts of _create_axes + # TODO(ArrayManager) HDFStore relies on accessing the blocks + if isinstance(frame._mgr, ArrayManager): + frame = frame._as_manager("block") + def get_blk_items(mgr): return [mgr.items.take(blk.mgr_locs) for blk in mgr.blocks] diff --git a/pandas/tests/io/pytables/test_append.py b/pandas/tests/io/pytables/test_append.py index 2569eb0c9e786..719b54a57a6c7 100644 --- a/pandas/tests/io/pytables/test_append.py +++ b/pandas/tests/io/pytables/test_append.py @@ -25,7 +25,7 @@ ensure_clean_store, ) -pytestmark = [pytest.mark.single, td.skip_array_manager_not_yet_implemented] +pytestmark = pytest.mark.single @pytest.mark.filterwarnings("ignore:object name:tables.exceptions.NaturalNameWarning") @@ -714,6 +714,10 @@ def check(obj, comparator): tm.assert_frame_equal(store.select("df2"), df) +# TODO(ArrayManager) currently we rely on falling back to BlockManager, but +# the conversion from AM->BM converts the invalid object dtype column into +# a datetime64 column no longer raising an error +@td.skip_array_manager_not_yet_implemented def test_append_raise(setup_path): with ensure_clean_store(setup_path) as store: diff --git a/pandas/tests/io/pytables/test_categorical.py b/pandas/tests/io/pytables/test_categorical.py index 0b3d56ebf959e..d2348ca8e314d 100644 --- a/pandas/tests/io/pytables/test_categorical.py +++ b/pandas/tests/io/pytables/test_categorical.py @@ -1,8 +1,6 @@ import numpy as np import pytest -import pandas.util._test_decorators as td - from pandas import ( Categorical, DataFrame, @@ -19,7 +17,6 @@ pytestmark = [ pytest.mark.single, - td.skip_array_manager_not_yet_implemented, # pytables https://github.com/PyTables/PyTables/issues/822 pytest.mark.filterwarnings( "ignore:a closed node found in the registry:UserWarning" diff --git a/pandas/tests/io/pytables/test_compat.py b/pandas/tests/io/pytables/test_compat.py index 4688d7d2be40a..c7200385aa998 100644 --- a/pandas/tests/io/pytables/test_compat.py +++ b/pandas/tests/io/pytables/test_compat.py @@ -1,15 +1,11 @@ import pytest -import pandas.util._test_decorators as td - import pandas as pd import pandas._testing as tm from pandas.tests.io.pytables.common import ensure_clean_path tables = pytest.importorskip("tables") -pytestmark = td.skip_array_manager_not_yet_implemented - @pytest.fixture def pytables_hdf5_file(): diff --git a/pandas/tests/io/pytables/test_complex.py b/pandas/tests/io/pytables/test_complex.py index 6cfe80ae5c87c..f3a43f669b1d5 100644 --- a/pandas/tests/io/pytables/test_complex.py +++ b/pandas/tests/io/pytables/test_complex.py @@ -3,8 +3,6 @@ import numpy as np import pytest -import pandas.util._test_decorators as td - import pandas as pd from pandas import ( DataFrame, @@ -18,9 +16,6 @@ from pandas.io.pytables import read_hdf -# TODO(ArrayManager) HDFStore relies on accessing the blocks -pytestmark = td.skip_array_manager_not_yet_implemented - def test_complex_fixed(setup_path): df = DataFrame( diff --git a/pandas/tests/io/pytables/test_errors.py b/pandas/tests/io/pytables/test_errors.py index 30b07fb572324..2ae330e5139be 100644 --- a/pandas/tests/io/pytables/test_errors.py +++ b/pandas/tests/io/pytables/test_errors.py @@ -6,8 +6,6 @@ import numpy as np import pytest -import pandas.util._test_decorators as td - from pandas import ( CategoricalIndex, DataFrame, @@ -27,7 +25,7 @@ _maybe_adjust_name, ) -pytestmark = [pytest.mark.single, td.skip_array_manager_not_yet_implemented] +pytestmark = pytest.mark.single def test_pass_spec_to_storer(setup_path): diff --git a/pandas/tests/io/pytables/test_file_handling.py b/pandas/tests/io/pytables/test_file_handling.py index 943b1bb06b1f3..88e2b5f080282 100644 --- a/pandas/tests/io/pytables/test_file_handling.py +++ b/pandas/tests/io/pytables/test_file_handling.py @@ -4,7 +4,6 @@ import pytest from pandas.compat import is_platform_little_endian -import pandas.util._test_decorators as td from pandas import ( DataFrame, @@ -27,7 +26,7 @@ Term, ) -pytestmark = [pytest.mark.single, td.skip_array_manager_not_yet_implemented] +pytestmark = pytest.mark.single def test_mode(setup_path): diff --git a/pandas/tests/io/pytables/test_keys.py b/pandas/tests/io/pytables/test_keys.py index 1dc2c9411ed7b..02b79bd0fdbc1 100644 --- a/pandas/tests/io/pytables/test_keys.py +++ b/pandas/tests/io/pytables/test_keys.py @@ -1,7 +1,5 @@ import pytest -import pandas.util._test_decorators as td - from pandas import ( DataFrame, HDFStore, @@ -13,7 +11,7 @@ tables, ) -pytestmark = [pytest.mark.single, td.skip_array_manager_not_yet_implemented] +pytestmark = pytest.mark.single def test_keys(setup_path): diff --git a/pandas/tests/io/pytables/test_put.py b/pandas/tests/io/pytables/test_put.py index 20278914a4838..4f8c7c84a9fcc 100644 --- a/pandas/tests/io/pytables/test_put.py +++ b/pandas/tests/io/pytables/test_put.py @@ -29,7 +29,7 @@ ) from pandas.util import _test_decorators as td -pytestmark = [pytest.mark.single, td.skip_array_manager_not_yet_implemented] +pytestmark = pytest.mark.single def test_format_type(setup_path): diff --git a/pandas/tests/io/pytables/test_pytables_missing.py b/pandas/tests/io/pytables/test_pytables_missing.py index fe474b7503e60..9adb0a6d227da 100644 --- a/pandas/tests/io/pytables/test_pytables_missing.py +++ b/pandas/tests/io/pytables/test_pytables_missing.py @@ -5,8 +5,6 @@ import pandas as pd import pandas._testing as tm -pytestmark = td.skip_array_manager_not_yet_implemented - @td.skip_if_installed("tables") def test_pytables_raises(): diff --git a/pandas/tests/io/pytables/test_read.py b/pandas/tests/io/pytables/test_read.py index 5d1deb45eba8b..1c9e63c66aadb 100644 --- a/pandas/tests/io/pytables/test_read.py +++ b/pandas/tests/io/pytables/test_read.py @@ -25,7 +25,7 @@ from pandas.io.pytables import TableIterator -pytestmark = [pytest.mark.single, td.skip_array_manager_not_yet_implemented] +pytestmark = pytest.mark.single def test_read_missing_key_close_store(setup_path): diff --git a/pandas/tests/io/pytables/test_retain_attributes.py b/pandas/tests/io/pytables/test_retain_attributes.py index c6e2904f7e670..16772d03c6d26 100644 --- a/pandas/tests/io/pytables/test_retain_attributes.py +++ b/pandas/tests/io/pytables/test_retain_attributes.py @@ -3,7 +3,6 @@ import pytest from pandas._libs.tslibs import Timestamp -import pandas.util._test_decorators as td from pandas import ( DataFrame, @@ -18,7 +17,7 @@ ensure_clean_store, ) -pytestmark = [pytest.mark.single, td.skip_array_manager_not_yet_implemented] +pytestmark = pytest.mark.single def test_retain_index_attributes(setup_path): diff --git a/pandas/tests/io/pytables/test_round_trip.py b/pandas/tests/io/pytables/test_round_trip.py index ce075943efe8a..97edc3cdffdf7 100644 --- a/pandas/tests/io/pytables/test_round_trip.py +++ b/pandas/tests/io/pytables/test_round_trip.py @@ -30,7 +30,7 @@ _default_compressor = "blosc" -pytestmark = [pytest.mark.single, td.skip_array_manager_not_yet_implemented] +pytestmark = pytest.mark.single def test_conv_read_write(setup_path): diff --git a/pandas/tests/io/pytables/test_select.py b/pandas/tests/io/pytables/test_select.py index fc19a3bd63c74..56d48945d5852 100644 --- a/pandas/tests/io/pytables/test_select.py +++ b/pandas/tests/io/pytables/test_select.py @@ -4,7 +4,6 @@ import pytest from pandas._libs.tslibs import Timestamp -import pandas.util._test_decorators as td import pandas as pd from pandas import ( @@ -28,7 +27,7 @@ from pandas.io.pytables import Term -pytestmark = [pytest.mark.single, td.skip_array_manager_not_yet_implemented] +pytestmark = pytest.mark.single def test_select_columns_in_where(setup_path): diff --git a/pandas/tests/io/pytables/test_store.py b/pandas/tests/io/pytables/test_store.py index c61864bbc0a76..856a2ca15ec4a 100644 --- a/pandas/tests/io/pytables/test_store.py +++ b/pandas/tests/io/pytables/test_store.py @@ -10,8 +10,6 @@ import numpy as np import pytest -import pandas.util._test_decorators as td - import pandas as pd from pandas import ( DataFrame, @@ -42,8 +40,7 @@ read_hdf, ) -# TODO(ArrayManager) HDFStore relies on accessing the blocks -pytestmark = [pytest.mark.single, td.skip_array_manager_not_yet_implemented] +pytestmark = pytest.mark.single def test_context(setup_path): diff --git a/pandas/tests/io/pytables/test_subclass.py b/pandas/tests/io/pytables/test_subclass.py index 05c9f0c650986..75b04f332e054 100644 --- a/pandas/tests/io/pytables/test_subclass.py +++ b/pandas/tests/io/pytables/test_subclass.py @@ -1,7 +1,5 @@ import numpy as np -import pandas.util._test_decorators as td - from pandas import ( DataFrame, Series, @@ -14,8 +12,6 @@ read_hdf, ) -pytestmark = td.skip_array_manager_not_yet_implemented - class TestHDFStoreSubclass: # GH 33748 diff --git a/pandas/tests/io/pytables/test_time_series.py b/pandas/tests/io/pytables/test_time_series.py index 181f63563665b..5e42dbde4b9f1 100644 --- a/pandas/tests/io/pytables/test_time_series.py +++ b/pandas/tests/io/pytables/test_time_series.py @@ -3,8 +3,6 @@ import numpy as np import pytest -import pandas.util._test_decorators as td - from pandas import ( DataFrame, Series, @@ -12,7 +10,7 @@ ) from pandas.tests.io.pytables.common import ensure_clean_store -pytestmark = [pytest.mark.single, td.skip_array_manager_not_yet_implemented] +pytestmark = pytest.mark.single def test_store_datetime_fractional_secs(setup_path): diff --git a/pandas/tests/io/pytables/test_timezones.py b/pandas/tests/io/pytables/test_timezones.py index 4aa6f94ca38e9..36fa79d0bb7e3 100644 --- a/pandas/tests/io/pytables/test_timezones.py +++ b/pandas/tests/io/pytables/test_timezones.py @@ -24,9 +24,6 @@ ensure_clean_store, ) -# TODO(ArrayManager) HDFStore relies on accessing the blocks -pytestmark = td.skip_array_manager_not_yet_implemented - def _compare_with_tz(a, b): tm.assert_frame_equal(a, b)