Documentation changes

Throws value error for older versions of pyarrow and if partitions_on is used in fastparquet

Author: anjsudh

pandas/io/parquet.py

@@ -113,6 +113,10 @@ def write(self, df, path, compression='snappy',
             self._validate_write_lt_070(df)
         path, _, _, _ = get_filepath_or_buffer(path, mode='wb')
 
+        if partition_cols is not None and self._pyarrow_lt_070:
+            raise ValueError("Partitioning of parquet files are only "
+                             "supported with pyarrow >= 0.7.0")
+
         if index is None:
             from_pandas_kwargs = {}
         else:
@@ -220,6 +224,9 @@ def write(self, df, path, compression='snappy', index=None,
         # DeprecationWarning: tostring() is deprecated.
         # Use tobytes() instead.
 
+        if 'partition_on' in kwargs:
+            raise ValueError("must use 'partition_cols' instead of 'partition_on'")
+
         if partition_cols is not None:
             kwargs['file_scheme'] = 'hive'
 

pandas/tests/io/test_parquet.py

@@ -496,8 +496,10 @@ def test_ignore_partition_cols_lt_070(self, pa_lt_070, df_full):
         partition_cols = ['bool', 'int']
         pa = pa_lt_070
         df = df_full
-        check_round_trip(df, pa,
-                         write_kwargs={'partition_cols': partition_cols})
+        with tm.ensure_clean_dir() as path:
+            df.to_parquet(path, partition_cols=partition_cols,
+                          compression=None)
+        pytest.raises(ValueError)
 
 
 class TestParquetFastParquet(Base):
@@ -576,3 +578,11 @@ def test_partition_cols_supported(self, fp, df_full):
             import fastparquet
             actual_partition_cols = fastparquet.ParquetFile(path, False).cats
             assert len(actual_partition_cols) == 2
+
+    def test_partition_on_not_supported(self, fp, df_full):
+        # GH #23283
+        partition_cols = ['bool', 'int']
+        df = df_full
+        with pytest.raises(ValueError):
+            df.to_parquet("", compression=None,
+              partition_on=partition_cols)

pandas/util/testing.py

@@ -778,15 +778,18 @@ def ensure_clean_dir():
     Get a temporary directory path and agrees to remove on close.
 
     Yields
-    ----------
+    ------
     Temporary directory path
     """
     directory_name = tempfile.mkdtemp(suffix='')
     try:
         yield directory_name
     finally:
-        import shutil
-        shutil.rmtree(directory_name)
+        try:
+            import shutil
+            shutil.rmtree(directory_name)
+        except Exception as e:
+            print("Exception on removing folder: {error}".format(error=e))
 
 
 # -----------------------------------------------------------------------------