File tree 1 file changed +17
-4
lines changed 1 file changed +17
-4
lines changed Original file line number Diff line number Diff line change 32
32
from pandas .core .dtypes .common import (
33
33
ensure_str ,
34
34
is_string_dtype ,
35
+ pandas_dtype ,
35
36
)
36
37
from pandas .core .dtypes .dtypes import PeriodDtype
37
38
38
39
from pandas import (
40
+ ArrowDtype ,
39
41
DataFrame ,
40
42
Index ,
41
43
MultiIndex ,
@@ -953,12 +955,23 @@ def _read_pyarrow(self) -> DataFrame:
953
955
Read JSON using the pyarrow engine.
954
956
"""
955
957
pyarrow_json = import_optional_dependency ("pyarrow.json" )
956
-
957
- pa_table = pyarrow_json .read_json (self .data )
958
- df = arrow_table_to_pandas (pa_table , dtype_backend = self .dtype_backend )
958
+ options = None
959
959
960
960
if isinstance (self .dtype , dict ):
961
- df = df .astype (self .dtype )
961
+ pa = import_optional_dependency ("pyarrow" )
962
+ fields = []
963
+ for field , dtype in self .dtype .items ():
964
+ pd_dtype = pandas_dtype (dtype )
965
+ if isinstance (pd_dtype , ArrowDtype ):
966
+ fields .append ((field , pd_dtype .pyarrow_dtype ))
967
+
968
+ schema = pa .schema (fields )
969
+ options = pyarrow_json .ParseOptions (
970
+ explicit_schema = schema , unexpected_field_behavior = "infer"
971
+ )
972
+
973
+ pa_table = pyarrow_json .read_json (self .data , parse_options = options )
974
+ df = arrow_table_to_pandas (pa_table , dtype_backend = self .dtype_backend )
962
975
963
976
return df
964
977
You can’t perform that action at this time.
0 commit comments