BUG: DataFrame.to_json OverflowError with np.long* dtypes

Author: gupta-paras

doc/source/whatsnew/v2.2.0.rst

@@ -348,6 +348,7 @@ I/O
 - Bug in :func:`read_csv` with ``engine="pyarrow"`` where ``usecols`` wasn't working with a csv with no headers (:issue:`54459`)
 - Bug in :func:`read_excel`, with ``engine="xlrd"`` (``xls`` files) erroring when file contains NaNs/Infs (:issue:`54564`)
 - Bug in :func:`to_excel`, with ``OdsWriter`` (``ods`` files) writing boolean/string value (:issue:`54994`)
+- Bug in :meth:`DataFrame.to_json` OverflowError with np.long* dtypes (:issue:`55403`)
 - Bug in :meth:`pandas.read_excel` with an ODS file without cached formatted cell for float values (:issue:`55219`)
 
 Period

pandas/_libs/include/pandas/vendored/ujson/lib/ultrajson.h

@@ -151,6 +151,7 @@ enum JSTYPES {
   JT_INVALID,  // Internal, do not return nor expect
   JT_POS_INF,  // Positive infinity
   JT_NEG_INF,  // Negative infinity
+  JT_LONGDOUBLE,  // (long double)
 };
 
 typedef void * JSOBJ;
@@ -182,6 +183,7 @@ typedef struct __JSONObjectEncoder {
   JSINT64 (*getLongValue)(JSOBJ obj, JSONTypeContext *tc);
   JSINT32 (*getIntValue)(JSOBJ obj, JSONTypeContext *tc);
   double (*getDoubleValue)(JSOBJ obj, JSONTypeContext *tc);
+  long double (*getLongDoubleValue)(JSOBJ obj, JSONTypeContext *tc);
   const char *(*getBigNumStringValue)(JSOBJ obj, JSONTypeContext *tc,
                                 size_t *_outLen);
 

pandas/_libs/src/vendored/ujson/lib/ultrajsonenc.c

@@ -74,7 +74,7 @@ The extra 2 bytes are for the quotes around the string
 */
 #define RESERVE_STRING(_len) (2 + ((_len)*6))
 
-static const double g_pow10[] = {1,
+static const long double g_pow10[] = {1,
                                  10,
                                  100,
                                  1000,
@@ -784,29 +784,29 @@ void Buffer_AppendLongUnchecked(JSONObjectEncoder *enc, JSINT64 value) {
     enc->offset += (wstr - (enc->offset));
 }
 
-int Buffer_AppendDoubleUnchecked(JSOBJ obj, JSONObjectEncoder *enc,
-                                 double value) {
+int Buffer_AppendLongDoubleUnchecked(JSOBJ obj, JSONObjectEncoder *enc,
+                                 long double value) {
     /* if input is beyond the thresholds, revert to exponential */
-    const double thres_max = (double)1e16 - 1;
-    const double thres_min = (double)1e-15;
+    const long double thres_max = (long double)1e16 - 1;
+    const long double thres_min = (long double)1e-15;
     char precision_str[20];
     int count;
-    double diff = 0.0;
+    long double diff = 0.0;
     char *str = enc->offset;
     char *wstr = str;
     unsigned long long whole;
-    double tmp;
+    long double tmp;
     unsigned long long frac;
     int neg;
-    double pow10;
+    long double pow10;
 
     if (value == HUGE_VAL || value == -HUGE_VAL) {
-        SetError(obj, enc, "Invalid Inf value when encoding double");
+        SetError(obj, enc, "Invalid Inf value when encoding long double");
         return FALSE;
     }
 
     if (!(value == value)) {
-        SetError(obj, enc, "Invalid Nan value when encoding double");
+        SetError(obj, enc, "Invalid Nan value when encoding long double");
         return FALSE;
     }
 
@@ -825,12 +825,12 @@ int Buffer_AppendDoubleUnchecked(JSOBJ obj, JSONObjectEncoder *enc,
         precision_str[0] = '%';
         precision_str[1] = '.';
 #if defined(_WIN32) && defined(_MSC_VER)
-        sprintf_s(precision_str + 2, sizeof(precision_str) - 2, "%ug",
+        sprintf_s(precision_str + 2, sizeof(precision_str) - 2, "%uLg",
                   enc->doublePrecision);
         enc->offset += sprintf_s(str, enc->end - enc->offset, precision_str,
                                  neg ? -value : value);
 #else
-        snprintf(precision_str + 2, sizeof(precision_str) - 2, "%ug",
+        snprintf(precision_str + 2, sizeof(precision_str) - 2, "%uLg",
                  enc->doublePrecision);
         enc->offset += snprintf(str, enc->end - enc->offset, precision_str,
                                 neg ? -value : value);
@@ -915,6 +915,7 @@ int Buffer_AppendDoubleUnchecked(JSOBJ obj, JSONObjectEncoder *enc,
     return TRUE;
 }
 
+
 /*
 FIXME:
 Handle integration functions returning NULL here */
@@ -942,7 +943,7 @@ void encode(JSOBJ obj, JSONObjectEncoder *enc, const char *name,
     This reservation must hold
 
     length of _name as encoded worst case +
-    maxLength of double to string OR maxLength of JSLONG to string
+    maxLength of long double to string OR maxLength of JSLONG to string
     */
 
     Buffer_Reserve(enc, 256 + RESERVE_STRING(cbName));
@@ -1077,8 +1078,18 @@ void encode(JSOBJ obj, JSONObjectEncoder *enc, const char *name,
         }
 
         case JT_DOUBLE: {
-            if (!Buffer_AppendDoubleUnchecked(obj, enc,
-                                              enc->getDoubleValue(obj, &tc))) {
+            if (!Buffer_AppendLongDoubleUnchecked(obj, enc,
+                                        (long double) enc->getDoubleValue(obj, &tc))) {
+                enc->endTypeContext(obj, &tc);
+                enc->level--;
+                return;
+            }
+            break;
+        }
+
+        case JT_LONGDOUBLE: {
+            if (!Buffer_AppendLongDoubleUnchecked(obj, enc,
+                                              enc->getLongDoubleValue(obj, &tc))) {
                 enc->endTypeContext(obj, &tc);
                 enc->level--;
                 return;

pandas/_libs/src/vendored/ujson/python/objToJSON.c

@@ -106,6 +106,7 @@ typedef struct __TypeContext {
     PyObject *iterator;
 
     double doubleValue;
+    long double longDoubleValue;
     JSINT64 longValue;
 
     char *cStr;
@@ -165,6 +166,7 @@ static TypeContext *createTypeContext(void) {
     pc->size = 0;
     pc->longValue = 0;
     pc->doubleValue = 0.0;
+    pc->longDoubleValue = 0.0L;
     pc->cStr = NULL;
     pc->npyarr = NULL;
     pc->pdblock = NULL;
@@ -1610,6 +1612,11 @@ void Object_beginTypeContext(JSOBJ _obj, JSONTypeContext *tc) {
                                   PyArray_DescrFromType(NPY_DOUBLE));
         tc->type = JT_DOUBLE;
         return;
+    } else if (PyArray_IsScalar(obj, LongDouble)) {
+        PyArray_CastScalarToCtype(obj, &(pc->longDoubleValue),
+                                  PyArray_DescrFromType(NPY_LONGDOUBLE));
+        tc->type = JT_LONGDOUBLE;
+        return;
     } else if (PyArray_Check(obj) && PyArray_CheckScalar(obj)) {
         PyErr_Format(PyExc_TypeError,
                      "%R (0d array) is not JSON serializable at the moment",
@@ -1929,6 +1936,10 @@ double Object_getDoubleValue(JSOBJ Py_UNUSED(obj), JSONTypeContext *tc) {
     return GET_TC(tc)->doubleValue;
 }
 
+double Object_getLongDoubleValue(JSOBJ Py_UNUSED(obj), JSONTypeContext *tc) {
+    return GET_TC(tc)->longDoubleValue;
+}
+
 const char *Object_getBigNumStringValue(JSOBJ obj, JSONTypeContext *tc,
                                         size_t *_outLen) {
     PyObject *repr = PyObject_Str(obj);
@@ -2007,6 +2018,7 @@ PyObject *objToJSON(PyObject *Py_UNUSED(self), PyObject *args,
         Object_getLongValue,
         NULL,  // getIntValue is unused
         Object_getDoubleValue,
+        Object_getLongDoubleValue,
         Object_getBigNumStringValue,
         Object_iterBegin,
         Object_iterNext,

pandas/tests/io/json/test_ujson.py

@@ -762,6 +762,14 @@ def test_float_array(self, float_numpy_dtype):
         )
         tm.assert_almost_equal(float_input, float_output)
 
+    def test_array_long_double(self):
+        dtype = np.longdouble
+        arr = np.arange(100.202, 200.202, 1, dtype=dtype)
+        arr = arr.reshape((5, 5, 4))
+
+        arr_out = np.array(ujson.ujson_loads(ujson.ujson_dumps(arr)), dtype=dtype)
+        tm.assert_almost_equal(arr, arr_out)
+
     def test_float_max(self, float_numpy_dtype):
         klass = np.dtype(float_numpy_dtype).type
         num = klass(np.finfo(float_numpy_dtype).max / 10)