ENH: fix arrow roundtrip for ExtensionDtypes in absence of pandas metadata (#34275)

Author: jorisvandenbossche

pandas/core/arrays/_arrow_utils.py

@@ -70,6 +70,11 @@ def __eq__(self, other):
         def __hash__(self):
             return hash((str(self), self.freq))
 
+        def to_pandas_dtype(self):
+            import pandas as pd
+
+            return pd.PeriodDtype(freq=self.freq)
+
     # register the type with a dummy instance
     _period_type = ArrowPeriodType("D")
     pyarrow.register_extension_type(_period_type)
@@ -119,6 +124,11 @@ def __eq__(self, other):
         def __hash__(self):
             return hash((str(self), str(self.subtype), self.closed))
 
+        def to_pandas_dtype(self):
+            import pandas as pd
+
+            return pd.IntervalDtype(self.subtype.to_pandas_dtype())
+
     # register the type with a dummy instance
     _interval_type = ArrowIntervalType(pyarrow.int64(), "left")
     pyarrow.register_extension_type(_interval_type)

pandas/tests/arrays/interval/test_interval.py

@@ -237,3 +237,26 @@ def test_arrow_table_roundtrip(breaks):
     result = table2.to_pandas()
     expected = pd.concat([df, df], ignore_index=True)
     tm.assert_frame_equal(result, expected)
+
+
+@pyarrow_skip
+@pytest.mark.parametrize(
+    "breaks",
+    [[0.0, 1.0, 2.0, 3.0], pd.date_range("2017", periods=4, freq="D")],
+    ids=["float", "datetime64[ns]"],
+)
+def test_arrow_table_roundtrip_without_metadata(breaks):
+    import pyarrow as pa
+
+    arr = IntervalArray.from_breaks(breaks)
+    arr[1] = None
+    df = pd.DataFrame({"a": arr})
+
+    table = pa.table(df)
+    # remove the metadata
+    table = table.replace_schema_metadata()
+    assert table.schema.metadata is None
+
+    result = table.to_pandas()
+    assert isinstance(result["a"].dtype, pd.IntervalDtype)
+    tm.assert_frame_equal(result, df)

pandas/tests/arrays/test_period.py

@@ -414,3 +414,21 @@ def test_arrow_table_roundtrip():
     result = table2.to_pandas()
     expected = pd.concat([df, df], ignore_index=True)
     tm.assert_frame_equal(result, expected)
+
+
+@pyarrow_skip
+def test_arrow_table_roundtrip_without_metadata():
+    import pyarrow as pa
+
+    arr = PeriodArray([1, 2, 3], freq="H")
+    arr[1] = pd.NaT
+    df = pd.DataFrame({"a": arr})
+
+    table = pa.table(df)
+    # remove the metadata
+    table = table.replace_schema_metadata()
+    assert table.schema.metadata is None
+
+    result = table.to_pandas()
+    assert isinstance(result["a"].dtype, PeriodDtype)
+    tm.assert_frame_equal(result, df)