diff --git a/pandas/io/tests/test_json/test_pandas.py b/pandas/io/tests/test_json/test_pandas.py index 2889ace..8ccbb93 100644 --- a/pandas/io/tests/test_json/test_pandas.py +++ b/pandas/io/tests/test_json/test_pandas.py @@ -815,10 +815,49 @@ DataFrame\\.index values are different \\(100\\.0 %\\) def test_default_handler(self): value = object() - frame = DataFrame({'a': ['a', value]}) - expected = frame.applymap(str) - result = pd.read_json(frame.to_json(default_handler=str)) + frame = DataFrame({'a': [7, value]}) + expected = DataFrame({'a': [7, value.id]}) + result = pd.read_json(frame.to_json(default_handler=lambda x: x.id)) assert_frame_equal(expected, result, check_index_type=False) + # Stringify by default. + expected = DataFrame({'a': [7, str(value)]}) + result = pd.read_json(frame.to_json()) + assert_frame_equal(expected, result, check_index_type=False) + + def test_default_handler_with_json_dumps(self): + import json + import sys + print >>sys.stderr, "Testing 1 two" + class PlausibleHandler(json.JSONEncoder): + # disable method-hidden because + # https://github.com/PyCQA/pylint/issues/414 + def default(self, obj): # pylint: disable=method-hidden + print >>sys.stderr, 'handling', obj + try: + if hasattr(obj, 'to_json'): + optargs = obj.to_json.__code__.co_varnames + if 'default_handler' in optargs: + print >>sys.stderr, 'Passing default_handler to to_json' + return obj.to_json(default_handler=self.default) + else: + print >>sys.stderr, 'Just calling to_json' + return obj.to_json() + elif isinstance(obj, complex): + return {'mathjs' : 'Complex', + 're' : x.real, 'im' : x.imag} + # https://github.com/pydata/pandas/issues/12554 + return json.JSONEncoder.default(self, obj) + except Exception as e: + return repr(e) + df_list = [ 9, DataFrame({'a': [1, 2.3, complex(4, -5)], + 'b': [float('nan'), None, 'N/A']})] + expected = ('[9,{"a":{"0":1.0,"1":2.3,' + '"2":{"mathjs":"Complex","re":4,"im":-5}},' + '"b":{"0":null,"1":null,"2":"N\/A"}}]') + self.assertEqual(expected, json.dumps(df_list, cls=PlausibleHandler)) + expected = ('{"a":{"0":1.0,"1":2.3,"2":"4-5j"},' + '"b":{"0":null,"1":null,"2":"N\/A"}}') + self.assertEqual(expected, df_list[1].to_json()) def test_default_handler_raises(self): def my_handler_raises(obj): diff --git a/pandas/src/ujson/lib/ultrajson.h b/pandas/src/ujson/lib/ultrajson.h index f83f74a..0149db8 100644 --- a/pandas/src/ujson/lib/ultrajson.h +++ b/pandas/src/ujson/lib/ultrajson.h @@ -155,6 +155,7 @@ enum JSTYPES JT_ARRAY, // Array structure JT_OBJECT, // Key/Value structure JT_INVALID, // Internal, do not return nor expect + JT_UNHANDLED, // Internal, do not return nor expect. Internally, try to use the defaultHandler if available. }; typedef void * JSOBJ; @@ -186,6 +187,12 @@ typedef struct __JSONObjectEncoder JSINT64 (*getLongValue)(JSOBJ obj, JSONTypeContext *tc); JSINT32 (*getIntValue)(JSOBJ obj, JSONTypeContext *tc); double (*getDoubleValue)(JSOBJ obj, JSONTypeContext *tc); + /* + Transform unhandled value in obj into something we can handle (like a string) in result. + Return a function pointer to free that object, which will be passed to it when finished, + or return NULL if transformation failed. + */ + JSPFN_FREE (*transformUnhandledValue)(JSOBJ obj, JSONTypeContext *tc, JSOBJ* result); /* Begin iteration of an iteratable object (JS_ARRAY or JS_OBJECT) diff --git a/pandas/src/ujson/lib/ultrajsonenc.c b/pandas/src/ujson/lib/ultrajsonenc.c index 5e2a226..507e50b 100644 --- a/pandas/src/ujson/lib/ultrajsonenc.c +++ b/pandas/src/ujson/lib/ultrajsonenc.c @@ -870,6 +870,20 @@ void encode(JSOBJ obj, JSONObjectEncoder *enc, const char *name, size_t cbName) Buffer_AppendCharUnchecked (enc, '\"'); break; } + + default: + { + JSPFN_FREE liberator; + JSOBJ result; + liberator = enc->transformUnhandledValue(obj, &tc, &result); + if (liberator) + { + encode (result, enc, NULL, 0); + liberator(result); + } + break; + } + } enc->endTypeContext(obj, &tc); diff --git a/pandas/src/ujson/python/objToJSON.c b/pandas/src/ujson/python/objToJSON.c index dcb509b..509f23b 100644 --- a/pandas/src/ujson/python/objToJSON.c +++ b/pandas/src/ujson/python/objToJSON.c @@ -158,8 +158,13 @@ enum PANDAS_FORMAT VALUES }; -//#define PRINTMARK() fprintf(stderr, "%s: MARK(%d)\n", __FILE__, __LINE__) -#define PRINTMARK() +// Debugging: uncomment the one you want. +// for gcc: +#define PRINTMARK() fprintf(stderr, "%s: MARK(%d)\t%s\n", __FILE__, __LINE__, __FUNCTION__) +// for c99: +//#define PRINTMARK() fprintf(stderr, "%s: MARK(%d)\t%s\n", __FILE__, __LINE__, __func__) +// for normal operation: +//#define PRINTMARK() int PdBlock_iterNext(JSOBJ, JSONTypeContext *); @@ -471,6 +476,46 @@ static void *PyTimeToJSON(JSOBJ _obj, JSONTypeContext *tc, void *outValue, size_ return outValue; } +static void PyUnhandledFree(JSOBJ obj) +{ + PRINTMARK(); + Py_DECREF((PyObject*)obj); +} +static JSPFN_FREE PyUnhandledTransformer(JSOBJ _obj, JSONTypeContext *tc, JSOBJ *_result) +{ + PyObject *obj = (PyObject *) _obj; + PyObjectEncoder *poenc = (PyObjectEncoder*) tc->encoder; + PyObject **result = (PyObject**) _result; + PyObject_Print(obj, stderr, 0); + PyObject_Print(poenc->enc.iterGetValue(_obj, tc), stderr, 0); + if (poenc->defaultHandler) + { + PRINTMARK(); + *result = PyObject_CallFunctionObjArgs(poenc->defaultHandler, obj, NULL); + } + else + { + PRINTMARK(); + *result = PyObject_Str(obj); + } + if (*result == NULL || PyErr_Occurred()) + { + PRINTMARK(); + if (!PyErr_Occurred()) + { + PyErr_SetString(PyExc_TypeError, "Failed to execute default handler"); + } + return NULL; + } + if (*result == obj) + { + PyErr_SetString(PyExc_TypeError, "Default handler returned same object as it received"); + return NULL; + } + PRINTMARK(); + return &PyUnhandledFree; +} + static int NpyTypeToJSONType(PyObject* obj, JSONTypeContext* tc, int npyType, void* value) { PyArray_VectorUnaryFunc* castfunc; @@ -544,12 +589,10 @@ static int NpyTypeToJSONType(PyObject* obj, JSONTypeContext* tc, int npyType, vo return *((npy_bool *) value) == NPY_TRUE ? JT_TRUE : JT_FALSE; } - PRINTMARK(); - PyErr_Format ( - PyExc_RuntimeError, - "Unhandled numpy dtype %d", - npyType); - return JT_INVALID; + PyObject_Print(value, stderr, 0); + PRINTMARK(); // GREMIO + PyObject_Print(obj, stderr, 0); + return JT_UNHANDLED; } @@ -2395,6 +2438,7 @@ ISITERABLE: PyErr_Clear(); + /* if (enc->defaultHandler) { PRINTMARK(); @@ -2411,6 +2455,7 @@ ISITERABLE: Py_DECREF(tmpObj); goto INVALID; } + */ PRINTMARK(); tc->type = JT_OBJECT; @@ -2531,6 +2576,7 @@ PyObject* objToJSON(PyObject* self, PyObject *args, PyObject *kwargs) Object_getLongValue, Object_getIntValue, Object_getDoubleValue, + PyUnhandledTransformer, Object_iterBegin, Object_iterNext, Object_iterEnd,