Write compliant Parquet with pyarrow if supported

Author: judahrand

pandas/io/parquet.py

@@ -13,6 +13,7 @@
     FilePathOrBuffer,
     StorageOptions,
 )
+from pandas.compat import pa_version_under4p0
 from pandas.compat._optional import import_optional_dependency
 from pandas.errors import AbstractMethodError
 from pandas.util._decorators import doc
@@ -179,6 +180,11 @@ def write(
             mode="wb",
             is_dir=partition_cols is not None,
         )
+
+        # Output compliant Parquet if PyArrow supports it
+        if not pa_version_under4p0:
+            kwargs["use_compliant_nested_type"] = True
+
         try:
             if partition_cols is not None:
                 # writes to multiple files under the given path
@@ -191,9 +197,7 @@ def write(
                 )
             else:
                 # write to single output file
-                self.api.parquet.write_table(
-                    table, path_or_handle, compression=compression, **kwargs
-                )
+                self.api.parquet.write_table(table, path_or_handle, compression=compression, **kwargs)
         finally:
             if handles is not None:
                 handles.close()