Fix issue #36271 - pd.read_json() fails for strings that look similar to fsspec_url

doc/source/whatsnew/v1.4.0.rst

@@ -664,6 +664,7 @@ I/O
 - Bug in :func:`read_csv` raising ``AttributeError`` when attempting to read a .csv file and infer index column dtype from an nullable integer type (:issue:`44079`)
 - :meth:`DataFrame.to_csv` and :meth:`Series.to_csv` with ``compression`` set to ``'zip'`` no longer create a zip file containing a file ending with ".zip". Instead, they try to infer the inner file name more smartly. (:issue:`39465`)
 - Bug in :func:`read_csv` when passing simultaneously a parser in ``date_parser`` and ``parse_dates=False``, the parsing was still called (:issue:`44366`)
+- Bug in :func:`read_json` raising ``ValueError`` when attempting to parse json strings containing urls (:issue:`36271`)
 
 Period
 ^^^^^^

pandas/io/common.py

@@ -18,6 +18,7 @@
 import mmap
 import os
 from pathlib import Path
+import re
 import tempfile
 from typing import (
     IO,
@@ -62,6 +63,7 @@
 
 _VALID_URLS = set(uses_relative + uses_netloc + uses_params)
 _VALID_URLS.discard("")
+_RFC_3986_PATTERN = re.compile(r"^[A-Za-z][A-Za-z0-9+\-+.]*://")
 
 BaseBufferT = TypeVar("BaseBufferT", bound=BaseBuffer)
 
@@ -247,7 +249,7 @@ def is_fsspec_url(url: FilePath | BaseBuffer) -> bool:
     """
     return (
         isinstance(url, str)
-        and "://" in url
+        and bool(_RFC_3986_PATTERN.match(url))
         and not url.startswith(("http://", "https://"))
     )
 

pandas/tests/io/json/test_pandas.py

@@ -1529,6 +1529,12 @@ def test_read_timezone_information(self):
         expected = Series([88], index=DatetimeIndex(["2019-01-01 11:00:00"], tz="UTC"))
         tm.assert_series_equal(result, expected)
 
+    def test_read_json_with_fsspec_value(self):
+        # GH 36271
+        result = read_json('{"url":{"0":"s3://example-fsspec"}}')
+        expected = DataFrame({"url": ["s3://example-fsspec"]})
+        tm.assert_frame_equal(result, expected)
+
     @pytest.mark.parametrize(
         "date_format,key", [("epoch", 86400000), ("iso", "P1DT0H0M0S")]
     )

pandas/tests/io/test_common.py

@@ -506,6 +506,11 @@ def test_is_fsspec_url():
     assert not icom.is_fsspec_url("random:pandas/somethingelse.com")
     assert not icom.is_fsspec_url("/local/path")
     assert not icom.is_fsspec_url("relative/local/path")
+    # fsspec URL in string should not be recognized
+    assert not icom.is_fsspec_url("this is not fsspec://url")
+    assert not icom.is_fsspec_url("{'url': 'gs://pandas/somethingelse.com'}")
+    # accept everything that conforms to RFC 3986 schema
+    assert icom.is_fsspec_url("RFC-3986+compliant.spec://something")
 
 
 @pytest.mark.parametrize("encoding", [None, "utf-8"])