fix to_json for numbers larger than sys.maxsize

doc/source/whatsnew/v1.1.0.rst

@@ -901,6 +901,7 @@ I/O
 - Bug in :meth:`~DataFrame.read_feather` was raising an `ArrowIOError` when reading an s3 or http file path (:issue:`29055`)
 - Bug in :meth:`~DataFrame.to_excel` could not handle the column name `render` and was raising an ``KeyError`` (:issue:`34331`)
 - Bug in :meth:`~SQLDatabase.execute` was raising a ``ProgrammingError`` for some DB-API drivers when the SQL statement contained the `%` character and no parameters were present (:issue:`34211`)
+- Bug in :meth:`json.dumps` was raising an `OverflowError` with numbers larger than sys.maxsize (:issue: `34395`)
 
 Plotting
 ^^^^^^^^

pandas/_libs/src/ujson/lib/ultrajson.h

@@ -150,6 +150,7 @@ enum JSTYPES {
   JT_INT,      // (JSINT32 (signed 32-bit))
   JT_LONG,     // (JSINT64 (signed 64-bit))
   JT_DOUBLE,   // (double)
+  JT_BIGNUM,   // integer larger than sys.maxsize
   JT_UTF8,     // (char 8-bit)
   JT_ARRAY,    // Array structure
   JT_OBJECT,   // Key/Value structure
@@ -187,6 +188,7 @@ typedef struct __JSONObjectEncoder {
   JSINT64 (*getLongValue)(JSOBJ obj, JSONTypeContext *tc);
   JSINT32 (*getIntValue)(JSOBJ obj, JSONTypeContext *tc);
   double (*getDoubleValue)(JSOBJ obj, JSONTypeContext *tc);
+  const char *(*getBigNumStringValue)(JSOBJ obj, JSONTypeContext *tc);
 
   /*
   Begin iteration of an iteratable object (JS_ARRAY or JS_OBJECT)

pandas/_libs/src/ujson/lib/ultrajsonenc.c

@@ -1107,6 +1107,27 @@ void encode(JSOBJ obj, JSONObjectEncoder *enc, const char *name,
             Buffer_AppendCharUnchecked(enc, '\"');
             break;
         }
+
+        case JT_BIGNUM: {
+
+            value = enc->getBigNumStringValue(obj, &tc);
+
+            Buffer_Reserve(enc, RESERVE_STRING(szlen));
+            if (enc->errorMsg) {
+                enc->endTypeContext(obj, &tc);
+                return;
+            }
+
+            if (!Buffer_EscapeStringValidated(obj, enc, value,
+                                                  value + szlen)) {
+                    enc->endTypeContext(obj, &tc);
+                    enc->level--;
+                    return;
+                }
+
+            break;
+
+        }
     }
 
     enc->endTypeContext(obj, &tc);

pandas/_libs/src/ujson/python/objToJSON.c

@@ -1635,7 +1635,9 @@ void Object_beginTypeContext(JSOBJ _obj, JSONTypeContext *tc) {
 
         if (exc && PyErr_ExceptionMatches(PyExc_OverflowError)) {
             PRINTMARK();
-            goto INVALID;
+            tc->type = JT_BIGNUM;
+            /* Question (arw2019): is this where I convert obj into a string 
+            and load result into tc->cStr? */
         }
 
         return;
@@ -2126,6 +2128,12 @@ double Object_getDoubleValue(JSOBJ Py_UNUSED(obj), JSONTypeContext *tc) {
     return GET_TC(tc)->doubleValue;
 }
 
+const char *Object_getBigNumStringValue(JSOBJ obj, JSONTypeContext *tc) {
+    // Question (arw2019): is this function right? 
+    // Do I need the outLen argument for example?
+    return GET_TC(tc)->cStr;
+}
+
 static void Object_releaseObject(JSOBJ _obj) { Py_DECREF((PyObject *)_obj); }
 
 void Object_iterBegin(JSOBJ obj, JSONTypeContext *tc) {

pandas/io/json/_json.py

@@ -27,6 +27,7 @@
 loads = json.loads
 dumps = json.dumps
 
+
 TABLE_SCHEMA_VERSION = "0.20.0"
 
 

pandas/tests/io/json/test_ujson.py

@@ -5,6 +5,7 @@
 import locale
 import math
 import re
+import sys
 import time
 
 import dateutil
@@ -559,6 +560,15 @@ def test_encode_long_conversion(self):
         assert output == json.dumps(long_input)
         assert long_input == ujson.decode(output)
 
+    def test_dumps_ints_larger_than_maxsize(self):
+        # GH34395
+        big_num = sys.maxsize + 1
+        encoding = ujson.dumps(big_num)
+
+        assert encoding == json.dumps(big_num)
+        # ujson.loads to be fixed in the future
+        # assert ujson.loads(encoding) == big_num
+
     @pytest.mark.parametrize(
         "int_exp", ["1337E40", "1.337E40", "1337E+9", "1.337e+40", "1.337E-4"]
     )