BUG: to_json not allowing uploads to S3 (pandas-dev#28375)

Author: YagoGG

doc/source/whatsnew/v1.0.1.rst

@@ -92,7 +92,6 @@ I/O
 
 - Fixed regression in :meth:`~DataFrame.to_csv` where specifying an ``na_rep`` might truncate the values written (:issue:`31447`)
 -
--
 
 Plotting
 ^^^^^^^^

doc/source/whatsnew/v1.1.0.rst

@@ -167,7 +167,7 @@ MultiIndex
 I/O
 ^^^
 - Bug in :meth:`read_json` where integer overflow was occuring when json contains big number strings. (:issue:`30320`)
--
+- Bug in :meth:`DataFrame.to_json` was raising ``NotFoundError`` when ``path_or_buf`` was an S3 URI (:issue:`28375`)
 -
 
 Plotting

pandas/core/generic.py

@@ -98,6 +98,7 @@
 from pandas.core.missing import find_valid_index
 from pandas.core.ops import _align_method_FRAME
 
+from pandas.io.common import get_filepath_or_buffer
 from pandas.io.formats import format as fmt
 from pandas.io.formats.format import DataFrameFormatter, format_percentiles
 from pandas.io.formats.printing import pprint_thing
@@ -2259,6 +2260,7 @@ def to_json(
         config.is_nonnegative_int(indent)
         indent = indent or 0
 
+
         return json.to_json(
             path_or_buf=path_or_buf,
             obj=self,

pandas/io/json/_json.py

@@ -56,7 +56,11 @@ def to_json(
             "'index=False' is only valid when 'orient' is 'split' or 'table'"
         )
 
-    path_or_buf = stringify_path(path_or_buf)
+    if path_or_buf is not None:
+        path_or_buf, _, _, _ = get_filepath_or_buffer(
+            path_or_buf, compression=compression, mode="w"
+        )
+
     if lines and orient != "records":
         raise ValueError("'lines' keyword only valid when 'orient' is records")
 

pandas/tests/io/json/test_pandas.py

@@ -1662,3 +1662,12 @@ def test_json_multiindex(self, dataframe, expected):
         series = dataframe.stack()
         result = series.to_json(orient="index")
         assert result == expected
+
+    def test_to_s3(self, s3_resource):
+        # GH 28375
+        mock_bucket_name, target_file = "pandas-test", "test.json"
+        df = DataFrame({"x": [1, 2, 3], "y": [2, 4, 6]})
+        df.to_json(f"s3://{mock_bucket_name}/{target_file}")
+        assert target_file in (
+            obj.key for obj in s3_resource.Bucket("pandas-test").objects.all()
+        )