[ArrayManager] Enable pytables IO by falling back to BlockManager (#41944)

Author: jorisvandenbossche

pandas/io/pytables.py

@@ -86,7 +86,10 @@
 )
 from pandas.core.construction import extract_array
 from pandas.core.indexes.api import ensure_index
-from pandas.core.internals import BlockManager
+from pandas.core.internals import (
+    ArrayManager,
+    BlockManager,
+)
 
 from pandas.io.common import stringify_path
 from pandas.io.formats.printing import (
@@ -3206,6 +3209,11 @@ def read(
 
     def write(self, obj, **kwargs):
         super().write(obj, **kwargs)
+
+        # TODO(ArrayManager) HDFStore relies on accessing the blocks
+        if isinstance(obj._mgr, ArrayManager):
+            obj = obj._as_manager("block")
+
         data = obj._mgr
         if not data.is_consolidated():
             data = data.consolidate()
@@ -4015,6 +4023,10 @@ def _get_blocks_and_items(
     ):
         # Helper to clarify non-state-altering parts of _create_axes
 
+        # TODO(ArrayManager) HDFStore relies on accessing the blocks
+        if isinstance(frame._mgr, ArrayManager):
+            frame = frame._as_manager("block")
+
         def get_blk_items(mgr):
             return [mgr.items.take(blk.mgr_locs) for blk in mgr.blocks]
 

pandas/tests/io/pytables/test_append.py

@@ -25,7 +25,7 @@
     ensure_clean_store,
 )
 
-pytestmark = [pytest.mark.single, td.skip_array_manager_not_yet_implemented]
+pytestmark = pytest.mark.single
 
 
 @pytest.mark.filterwarnings("ignore:object name:tables.exceptions.NaturalNameWarning")
@@ -714,6 +714,10 @@ def check(obj, comparator):
         tm.assert_frame_equal(store.select("df2"), df)
 
 
+# TODO(ArrayManager) currently we rely on falling back to BlockManager, but
+# the conversion from AM->BM converts the invalid object dtype column into
+# a datetime64 column no longer raising an error
+@td.skip_array_manager_not_yet_implemented
 def test_append_raise(setup_path):
 
     with ensure_clean_store(setup_path) as store:

pandas/tests/io/pytables/test_categorical.py

@@ -1,8 +1,6 @@
 import numpy as np
 import pytest
 
-import pandas.util._test_decorators as td
-
 from pandas import (
     Categorical,
     DataFrame,
@@ -19,7 +17,6 @@
 
 pytestmark = [
     pytest.mark.single,
-    td.skip_array_manager_not_yet_implemented,
     # pytables https://github.com/PyTables/PyTables/issues/822
     pytest.mark.filterwarnings(
         "ignore:a closed node found in the registry:UserWarning"

pandas/tests/io/pytables/test_compat.py

@@ -1,15 +1,11 @@
 import pytest
 
-import pandas.util._test_decorators as td
-
 import pandas as pd
 import pandas._testing as tm
 from pandas.tests.io.pytables.common import ensure_clean_path
 
 tables = pytest.importorskip("tables")
 
-pytestmark = td.skip_array_manager_not_yet_implemented
-
 
 @pytest.fixture
 def pytables_hdf5_file():

pandas/tests/io/pytables/test_complex.py

@@ -3,8 +3,6 @@
 import numpy as np
 import pytest
 
-import pandas.util._test_decorators as td
-
 import pandas as pd
 from pandas import (
     DataFrame,
@@ -18,9 +16,6 @@
 
 from pandas.io.pytables import read_hdf
 
-# TODO(ArrayManager) HDFStore relies on accessing the blocks
-pytestmark = td.skip_array_manager_not_yet_implemented
-
 
 def test_complex_fixed(setup_path):
     df = DataFrame(

pandas/tests/io/pytables/test_errors.py

@@ -6,8 +6,6 @@
 import numpy as np
 import pytest
 
-import pandas.util._test_decorators as td
-
 from pandas import (
     CategoricalIndex,
     DataFrame,
@@ -27,7 +25,7 @@
     _maybe_adjust_name,
 )
 
-pytestmark = [pytest.mark.single, td.skip_array_manager_not_yet_implemented]
+pytestmark = pytest.mark.single
 
 
 def test_pass_spec_to_storer(setup_path):

pandas/tests/io/pytables/test_file_handling.py

@@ -4,7 +4,6 @@
 import pytest
 
 from pandas.compat import is_platform_little_endian
-import pandas.util._test_decorators as td
 
 from pandas import (
     DataFrame,
@@ -27,7 +26,7 @@
     Term,
 )
 
-pytestmark = [pytest.mark.single, td.skip_array_manager_not_yet_implemented]
+pytestmark = pytest.mark.single
 
 
 def test_mode(setup_path):

pandas/tests/io/pytables/test_keys.py

@@ -1,7 +1,5 @@
 import pytest
 
-import pandas.util._test_decorators as td
-
 from pandas import (
     DataFrame,
     HDFStore,
@@ -13,7 +11,7 @@
     tables,
 )
 
-pytestmark = [pytest.mark.single, td.skip_array_manager_not_yet_implemented]
+pytestmark = pytest.mark.single
 
 
 def test_keys(setup_path):

pandas/tests/io/pytables/test_put.py

@@ -29,7 +29,7 @@
 )
 from pandas.util import _test_decorators as td
 
-pytestmark = [pytest.mark.single, td.skip_array_manager_not_yet_implemented]
+pytestmark = pytest.mark.single
 
 
 def test_format_type(setup_path):

pandas/tests/io/pytables/test_pytables_missing.py

@@ -5,8 +5,6 @@
 import pandas as pd
 import pandas._testing as tm
 
-pytestmark = td.skip_array_manager_not_yet_implemented
-
 
 @td.skip_if_installed("tables")
 def test_pytables_raises():

pandas/tests/io/pytables/test_read.py

@@ -25,7 +25,7 @@
 
 from pandas.io.pytables import TableIterator
 
-pytestmark = [pytest.mark.single, td.skip_array_manager_not_yet_implemented]
+pytestmark = pytest.mark.single
 
 
 def test_read_missing_key_close_store(setup_path):

pandas/tests/io/pytables/test_retain_attributes.py

@@ -3,7 +3,6 @@
 import pytest
 
 from pandas._libs.tslibs import Timestamp
-import pandas.util._test_decorators as td
 
 from pandas import (
     DataFrame,
@@ -18,7 +17,7 @@
     ensure_clean_store,
 )
 
-pytestmark = [pytest.mark.single, td.skip_array_manager_not_yet_implemented]
+pytestmark = pytest.mark.single
 
 
 def test_retain_index_attributes(setup_path):

pandas/tests/io/pytables/test_round_trip.py

@@ -30,7 +30,7 @@
 _default_compressor = "blosc"
 
 
-pytestmark = [pytest.mark.single, td.skip_array_manager_not_yet_implemented]
+pytestmark = pytest.mark.single
 
 
 def test_conv_read_write(setup_path):

pandas/tests/io/pytables/test_select.py

@@ -4,7 +4,6 @@
 import pytest
 
 from pandas._libs.tslibs import Timestamp
-import pandas.util._test_decorators as td
 
 import pandas as pd
 from pandas import (
@@ -28,7 +27,7 @@
 
 from pandas.io.pytables import Term
 
-pytestmark = [pytest.mark.single, td.skip_array_manager_not_yet_implemented]
+pytestmark = pytest.mark.single
 
 
 def test_select_columns_in_where(setup_path):

pandas/tests/io/pytables/test_store.py

@@ -10,8 +10,6 @@
 import numpy as np
 import pytest
 
-import pandas.util._test_decorators as td
-
 import pandas as pd
 from pandas import (
     DataFrame,
@@ -42,8 +40,7 @@
     read_hdf,
 )
 
-# TODO(ArrayManager) HDFStore relies on accessing the blocks
-pytestmark = [pytest.mark.single, td.skip_array_manager_not_yet_implemented]
+pytestmark = pytest.mark.single
 
 
 def test_context(setup_path):

pandas/tests/io/pytables/test_subclass.py

@@ -1,7 +1,5 @@
 import numpy as np
 
-import pandas.util._test_decorators as td
-
 from pandas import (
     DataFrame,
     Series,
@@ -14,8 +12,6 @@
     read_hdf,
 )
 
-pytestmark = td.skip_array_manager_not_yet_implemented
-
 
 class TestHDFStoreSubclass:
     # GH 33748

pandas/tests/io/pytables/test_time_series.py

@@ -3,16 +3,14 @@
 import numpy as np
 import pytest
 
-import pandas.util._test_decorators as td
-
 from pandas import (
     DataFrame,
     Series,
     _testing as tm,
 )
 from pandas.tests.io.pytables.common import ensure_clean_store
 
-pytestmark = [pytest.mark.single, td.skip_array_manager_not_yet_implemented]
+pytestmark = pytest.mark.single
 
 
 def test_store_datetime_fractional_secs(setup_path):

pandas/tests/io/pytables/test_timezones.py

@@ -24,9 +24,6 @@
     ensure_clean_store,
 )
 
-# TODO(ArrayManager) HDFStore relies on accessing the blocks
-pytestmark = td.skip_array_manager_not_yet_implemented
-
 
 def _compare_with_tz(a, b):
     tm.assert_frame_equal(a, b)