fix to_json for numbers larger than sys.maxsize

doc/source/whatsnew/v1.1.0.rst

@@ -901,6 +901,7 @@ I/O
 - Bug in :meth:`~DataFrame.read_feather` was raising an `ArrowIOError` when reading an s3 or http file path (:issue:`29055`)
 - Bug in :meth:`~DataFrame.to_excel` could not handle the column name `render` and was raising an ``KeyError`` (:issue:`34331`)
 - Bug in :meth:`~SQLDatabase.execute` was raising a ``ProgrammingError`` for some DB-API drivers when the SQL statement contained the `%` character and no parameters were present (:issue:`34211`)
+- Bug in :meth:`json.dumps` was raising an `OverflowError` with numbers larger than sys.maxsize (:issue: `34395`)
 
 Plotting
 ^^^^^^^^

pandas/_libs/src/ujson/lib/ultrajson.h

@@ -150,6 +150,7 @@ enum JSTYPES {
   JT_INT,      // (JSINT32 (signed 32-bit))
   JT_LONG,     // (JSINT64 (signed 64-bit))
   JT_DOUBLE,   // (double)
+  JT_BIGNUM,   // integer larger than sys.maxsize
   JT_UTF8,     // (char 8-bit)
   JT_ARRAY,    // Array structure
   JT_OBJECT,   // Key/Value structure
@@ -187,6 +188,7 @@ typedef struct __JSONObjectEncoder {
   JSINT64 (*getLongValue)(JSOBJ obj, JSONTypeContext *tc);
   JSINT32 (*getIntValue)(JSOBJ obj, JSONTypeContext *tc);
   double (*getDoubleValue)(JSOBJ obj, JSONTypeContext *tc);
+  const char *(*getBigNumStringValue)(JSOBJ obj, JSONTypeContext *tc);
 
   /*
   Begin iteration of an iteratable object (JS_ARRAY or JS_OBJECT)

pandas/_libs/src/ujson/lib/ultrajsonenc.c

@@ -1107,6 +1107,27 @@ void encode(JSOBJ obj, JSONObjectEncoder *enc, const char *name,
             Buffer_AppendCharUnchecked(enc, '\"');
             break;
         }
+
+        case JT_BIGNUM: {
+
+            value = enc->getBigNumStringValue(obj, &tc);
+
+            Buffer_Reserve(enc, RESERVE_STRING(szlen));
+            if (enc->errorMsg) {
+                enc->endTypeContext(obj, &tc);
+                return;
+            }
+
+            if (!Buffer_EscapeStringValidated(obj, enc, value,
+                                                  value + szlen)) {
+                    enc->endTypeContext(obj, &tc);
+                    enc->level--;
+                    return;
+                }
+
+            break;
+
+        }
     }
 
     enc->endTypeContext(obj, &tc);

pandas/_libs/src/ujson/python/objToJSON.c

@@ -1635,7 +1635,10 @@ void Object_beginTypeContext(JSOBJ _obj, JSONTypeContext *tc) {
 
         if (exc && PyErr_ExceptionMatches(PyExc_OverflowError)) {
             PRINTMARK();
-            goto INVALID;
+            tc->type = JT_BIGNUM;
+
+            // This line generates compiler errors.
+            GET_TC(tc)->cStr = Object_getBigNumStringValue(obj);
         }
 
         return;
@@ -2126,6 +2129,41 @@ double Object_getDoubleValue(JSOBJ Py_UNUSED(obj), JSONTypeContext *tc) {
     return GET_TC(tc)->doubleValue;
 }
 
+// /*
+const char *Object_getBigNumStringValue(JSOBJ obj, JSONTypeContext *tc, 
+                                    size_t *_outLen) {
+    /* here goes the code that converts obj into a string
+    char *wstr;
+    if (obj<0) {wstr++ = "-";}
+    int digit;
+    PyObject rem;
+    PyObject ten;
+    long ten_as_long = 10, rem_as_long;
+    ten = PyNumber_FromLong(ten_as_long);
+    do {
+        rem = PyNumber_Remainder(obj, ten);
+        obj = PyNumber_FloorDivide(obj, ten);
+
+        rem_as_long = PyLong_AsLong(rem);
+        wstr++ = char(48 + (int) rem_as_long);
+    } while (obj>10);
+    */
+
+    // we then load that string into tc->cStr
+    GET_TC(tc)->str = wstr;
+
+    /* _outLen: do we set that here?
+    I can imagine counting the number of digits in the 
+    do-while loop and then setting
+    _outLen = (number of digits);
+    I'm not quite sure how that would work though
+    since _outLen is an argument to this function
+    */
+
+    return GET_TC(tc)->cStr;
+}
+// */
+
 static void Object_releaseObject(JSOBJ _obj) { Py_DECREF((PyObject *)_obj); }
 
 void Object_iterBegin(JSOBJ obj, JSONTypeContext *tc) {
@@ -2179,6 +2217,7 @@ PyObject *objToJSON(PyObject *Py_UNUSED(self), PyObject *args,
         Object_endTypeContext,
         Object_getStringValue,
         Object_getLongValue,
+        Object_getBigNumStringValue,
         NULL, // getIntValue is unused
         Object_getDoubleValue,
         Object_iterBegin,

pandas/io/json/_json.py

@@ -27,6 +27,7 @@
 loads = json.loads
 dumps = json.dumps
 
+
 TABLE_SCHEMA_VERSION = "0.20.0"
 
 

pandas/tests/io/json/test_ujson.py

@@ -5,6 +5,7 @@
 import locale
 import math
 import re
+import sys
 import time
 
 import dateutil
@@ -559,6 +560,15 @@ def test_encode_long_conversion(self):
         assert output == json.dumps(long_input)
         assert long_input == ujson.decode(output)
 
+    def test_dumps_ints_larger_than_maxsize(self):
+        # GH34395
+        big_num = sys.maxsize + 1
+        encoding = ujson.dumps(big_num)
+
+        assert str(encoding) == json.dumps(big_num)
+        # ujson.loads to be fixed in the future
+        # assert ujson.loads(encoding) == big_num
+
     @pytest.mark.parametrize(
         "int_exp", ["1337E40", "1.337E40", "1337E+9", "1.337e+40", "1.337E-4"]
     )