From d4b6c7201333c34b5ace3715a9a8cfd47a3dd712 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Fri, 13 Dec 2019 19:07:35 -0800 Subject: [PATCH 01/14] new implementations --- pandas/_libs/src/ujson/python/objToJSON.c | 115 ++++++++++------------ 1 file changed, 51 insertions(+), 64 deletions(-) diff --git a/pandas/_libs/src/ujson/python/objToJSON.c b/pandas/_libs/src/ujson/python/objToJSON.c index 5d17d3a2d7bcb..f01525ede6177 100644 --- a/pandas/_libs/src/ujson/python/objToJSON.c +++ b/pandas/_libs/src/ujson/python/objToJSON.c @@ -408,80 +408,67 @@ static void *PyUnicodeToUTF8(JSOBJ _obj, JSONTypeContext *tc, void *outValue, return PyUnicode_AsUTF8AndSize(_obj, _outLen); } -static void *PandasDateTimeStructToJSON(npy_datetimestruct *dts, - JSONTypeContext *tc, void *outValue, - size_t *_outLen) { - NPY_DATETIMEUNIT base = ((PyObjectEncoder *)tc->encoder)->datetimeUnit; - if (((PyObjectEncoder *)tc->encoder)->datetimeIso) { - PRINTMARK(); - *_outLen = (size_t)get_datetime_iso_8601_strlen(0, base); - GET_TC(tc)->cStr = PyObject_Malloc(sizeof(char) * (*_outLen)); - if (!GET_TC(tc)->cStr) { - PyErr_NoMemory(); - ((JSONObjectEncoder *)tc->encoder)->errorMsg = ""; - return NULL; - } - - if (!make_iso_8601_datetime(dts, GET_TC(tc)->cStr, *_outLen, base)) { - PRINTMARK(); - *_outLen = strlen(GET_TC(tc)->cStr); - return GET_TC(tc)->cStr; - } else { - PRINTMARK(); - PyErr_SetString(PyExc_ValueError, - "Could not convert datetime value to string"); - ((JSONObjectEncoder *)tc->encoder)->errorMsg = ""; - PyObject_Free(GET_TC(tc)->cStr); - return NULL; - } - } else { - PRINTMARK(); - *((JSINT64 *)outValue) = npy_datetimestruct_to_datetime(base, dts); - return NULL; - } -} +/* returns a char* and mutates the pointer to *len */ +static char *NpyDateTimeToIso(npy_datetime dt, JSONTypeContext *tc, size_t *len) { + npy_datetimestruct dts; + NPY_DATETIMEUNIT base = ((PyObjectEncoder *)tc->encoder)->datetimeUnit; + int ret_code; + + pandas_datetime_to_datetimestruct(dt, NPY_FR_NS, &dts); + *len = (size_t)get_datetime_iso_8601_strlen(0, base); + char *result = PyObject_Malloc(*len); + + if (result == NULL) { + PyErr_NoMemory(); + ((JSONObjectEncoder *)tc->encoder)->errorMsg = ""; + return NULL; + } -static void *NpyDateTimeScalarToJSON(JSOBJ _obj, JSONTypeContext *tc, - void *outValue, size_t *_outLen) { - npy_datetimestruct dts; - PyDatetimeScalarObject *obj = (PyDatetimeScalarObject *)_obj; - PRINTMARK(); - // TODO(anyone): Does not appear to be reached in tests. + ret_code = make_iso_8601_datetime(&dts, result, *len, base); + if (ret_code != 0) { + PyErr_SetString(PyExc_ValueError, + "Could not convert datetime value to string"); + ((JSONObjectEncoder *)tc->encoder)->errorMsg = ""; + PyObject_Free(result); + } - pandas_datetime_to_datetimestruct(obj->obval, - (NPY_DATETIMEUNIT)obj->obmeta.base, &dts); - return PandasDateTimeStructToJSON(&dts, tc, outValue, _outLen); + return result; } -static void *PyDateTimeToJSON(JSOBJ _obj, JSONTypeContext *tc, void *outValue, - size_t *_outLen) { - npy_datetimestruct dts; - PyDateTime_Date *obj = (PyDateTime_Date *)_obj; - - PRINTMARK(); +static char *PyDateTimeToIso(PyObject *obj, JSONTypeContext *tc, size_t *len) { + npy_datetimestruct dts; + int ret; + + if (!PyDateTime_Check(obj)) { + // TODO: raise TypeError + } - if (!convert_pydatetime_to_datetimestruct(obj, &dts)) { - PRINTMARK(); - return PandasDateTimeStructToJSON(&dts, tc, outValue, _outLen); - } else { - if (!PyErr_Occurred()) { - PyErr_SetString(PyExc_ValueError, - "Could not convert datetime value to string"); - } - ((JSONObjectEncoder *)tc->encoder)->errorMsg = ""; - return NULL; + ret = convert_pydatetime_to_datetimestruct(obj, &dts); + if (ret != 0) { + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ValueError, + "Could not convert datetime value to string"); } -} + ((JSONObjectEncoder *)tc->encoder)->errorMsg = ""; + return NULL; + } -static void *NpyDatetime64ToJSON(JSOBJ _obj, JSONTypeContext *tc, - void *outValue, size_t *_outLen) { - npy_datetimestruct dts; + NPY_DATETIMEUNIT base = ((PyObjectEncoder *)tc->encoder)->datetimeUnit; + *len = (size_t)get_datetime_iso_8601_strlen(0, base); + char *result = PyObject_Malloc(*len); + ret = make_iso_8601_datetime(&dts, result, *len, base); + + if (ret != 0) { PRINTMARK(); + PyErr_SetString(PyExc_ValueError, + "Could not convert datetime value to string"); + ((JSONObjectEncoder *)tc->encoder)->errorMsg = ""; + PyObject_Free(result); + return NULL; + } - pandas_datetime_to_datetimestruct((npy_datetime)GET_TC(tc)->longValue, - NPY_FR_ns, &dts); - return PandasDateTimeStructToJSON(&dts, tc, outValue, _outLen); + return result; } static void *PyTimeToJSON(JSOBJ _obj, JSONTypeContext *tc, void *outValue, From d2c6d367a111a4c6c98b07d6007cd80de1ce1eaf Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sun, 15 Dec 2019 10:40:00 -0800 Subject: [PATCH 02/14] Stop point --- pandas/_libs/src/ujson/python/objToJSON.c | 126 +++++++++------------- 1 file changed, 53 insertions(+), 73 deletions(-) diff --git a/pandas/_libs/src/ujson/python/objToJSON.c b/pandas/_libs/src/ujson/python/objToJSON.c index f01525ede6177..4f9dd40623ed5 100644 --- a/pandas/_libs/src/ujson/python/objToJSON.c +++ b/pandas/_libs/src/ujson/python/objToJSON.c @@ -410,12 +410,13 @@ static void *PyUnicodeToUTF8(JSOBJ _obj, JSONTypeContext *tc, void *outValue, /* returns a char* and mutates the pointer to *len */ -static char *NpyDateTimeToIso(npy_datetime dt, JSONTypeContext *tc, size_t *len) { +static char *NpyDateTimeToIso(npy_datetime dt, JSONTypeContext *tc, void *unused, + size_t *len) { npy_datetimestruct dts; NPY_DATETIMEUNIT base = ((PyObjectEncoder *)tc->encoder)->datetimeUnit; int ret_code; - pandas_datetime_to_datetimestruct(dt, NPY_FR_NS, &dts); + pandas_datetime_to_datetimestruct(dt, base, &dts); *len = (size_t)get_datetime_iso_8601_strlen(0, base); char *result = PyObject_Malloc(*len); @@ -436,7 +437,13 @@ static char *NpyDateTimeToIso(npy_datetime dt, JSONTypeContext *tc, size_t *len) return result; } -static char *PyDateTimeToIso(PyObject *obj, JSONTypeContext *tc, size_t *len) { +static npy_datetime NpyDateTimeToEpoch(npy_datetime dt, NPY_DATETIMEUNIT base) { + scaleNanosecToUnit(&dt, base); + return dt; +} + +static char *PyDateTimeToIso(PyObject *obj, JSONTypeContext *tc, void *unused, + size_t *len) { npy_datetimestruct dts; int ret; @@ -448,7 +455,7 @@ static char *PyDateTimeToIso(PyObject *obj, JSONTypeContext *tc, size_t *len) { if (ret != 0) { if (!PyErr_Occurred()) { PyErr_SetString(PyExc_ValueError, - "Could not convert datetime value to string"); + "Could not convert PyDateTime to numpy datetime"); } ((JSONObjectEncoder *)tc->encoder)->errorMsg = ""; return NULL; @@ -471,6 +478,29 @@ static char *PyDateTimeToIso(PyObject *obj, JSONTypeContext *tc, size_t *len) { return result; } +static npy_datetime PyDateTimeToEpoch(PyObject *obj, NPY_DATETIMEUNIT base) { + npy_datetimestruct dts; + int ret; + + if (!PyDateTime_Check(obj)) { + // TODO: raise TypeError + } + + ret = convert_pydatetime_to_datetimestruct(obj, &dts); + if (ret != 0) { + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ValueError, + "Could not convert PyDateTime to numpy datetime"); + } + // TODO: is setting errMsg required? + //((JSONObjectEncoder *)tc->encoder)->errorMsg = ""; + return NULL; + } + + npy_datetime npy_dt = npy_datetimestruct_to_datetime(base, &dts); + return NpyDateTimeToEpoch(npy_dt, base); +} + static void *PyTimeToJSON(JSOBJ _obj, JSONTypeContext *tc, void *outValue, size_t *outLen) { PyObject *obj = (PyObject *)_obj; @@ -500,45 +530,6 @@ static void *PyTimeToJSON(JSOBJ _obj, JSONTypeContext *tc, void *outValue, return outValue; } -static int NpyTypeToJSONType(PyObject *obj, JSONTypeContext *tc, int npyType, - void *value) { - PyArray_VectorUnaryFunc *castfunc; - npy_int64 longVal; - - if (PyTypeNum_ISDATETIME(npyType)) { - PRINTMARK(); - castfunc = - PyArray_GetCastFunc(PyArray_DescrFromType(npyType), NPY_INT64); - if (!castfunc) { - PyErr_Format(PyExc_ValueError, "Cannot cast numpy dtype %d to long", - npyType); - } - castfunc(value, &longVal, 1, NULL, NULL); - if (longVal == get_nat()) { - PRINTMARK(); - return JT_NULL; - } - - if (((PyObjectEncoder *)tc->encoder)->datetimeIso) { - GET_TC(tc)->longValue = (JSINT64)longVal; - GET_TC(tc)->PyTypeToJSON = NpyDatetime64ToJSON; - return JT_UTF8; - } else { - NPY_DATETIMEUNIT unit = - ((PyObjectEncoder *)tc->encoder)->datetimeUnit; - if (!scaleNanosecToUnit(&longVal, unit)) { - GET_TC(tc)->longValue = longVal; - return JT_LONG; - } else { - // TODO: some kind of error handling - } - } - } - - PRINTMARK(); - return JT_INVALID; -} - //============================================================================= // Numpy array iteration functions //============================================================================= @@ -1569,7 +1560,7 @@ char **NpyArr_encodeLabels(PyArrayObject *labels, PyObjectEncoder *enc, cLabel = (char *)PyUnicode_AsUTF8(iso); Py_DECREF(iso); len = strlen(cLabel); - } else if (PyTypeNum_ISDATETIME(type_num) || PyDateTime_Check(item) || + } else if (PyTypeNum_ISDATETIME(enc->npyType) || PyDateTime_Check(item) || PyDate_Check(item)) { PyObject *ts = PyObject_CallFunction(cls_timestamp, "(O)", item); if (ts == NULL) { @@ -1692,29 +1683,6 @@ void Object_beginTypeContext(JSOBJ _obj, JSONTypeContext *tc) { obj = (PyObject *)_obj; enc = (PyObjectEncoder *)tc->encoder; - if (enc->npyType >= 0) { - PRINTMARK(); - tc->prv = &(enc->basicTypeContext); - tc->type = NpyTypeToJSONType(obj, tc, enc->npyType, enc->npyValue); - - if (tc->type == JT_INVALID) { - if (enc->defaultHandler) { - enc->npyType = -1; - PRINTMARK(); - Object_invokeDefaultHandler( - enc->npyCtxtPassthru->getitem(enc->npyValue, - enc->npyCtxtPassthru->array), - enc); - } else { - PyErr_Format(PyExc_RuntimeError, "Unhandled numpy dtype %d", - enc->npyType); - } - } - enc->npyCtxtPassthru = NULL; - enc->npyType = -1; - return; - } - if (PyBool_Check(obj)) { PRINTMARK(); tc->type = (obj == Py_True) ? JT_TRUE : JT_FALSE; @@ -1776,6 +1744,19 @@ void Object_beginTypeContext(JSOBJ _obj, JSONTypeContext *tc) { GET_TC(tc)->doubleValue = PyFloat_AsDouble(obj); tc->type = JT_DOUBLE; return; + } else if (PyTypeNum_ISDATETIME(enc->npyType)) { + PRINTMARK(); + if (enc->datetimeIso) { + PRINTMARK(); + pc->PyTypeToJSON = NpyDateTimeToIso; + tc->type = JT_UTF8; + } else { + PRINTMARK(); + NPY_DATETIMEUNIT base = ((PyObjectEncoder *)tc->encoder)->datetimeUnit; + GET_TC(tc)->longValue = NpyDateTimeToEpoch(obj, base); + tc->type = JT_LONG; + } + return; } else if (PyDateTime_Check(obj) || PyDate_Check(obj)) { if (PyObject_TypeCheck(obj, cls_nat)) { PRINTMARK(); @@ -1786,13 +1767,12 @@ void Object_beginTypeContext(JSOBJ _obj, JSONTypeContext *tc) { PRINTMARK(); if (enc->datetimeIso) { PRINTMARK(); - pc->PyTypeToJSON = PyDateTimeToJSON; + pc->PyTypeToJSON = PyDateTimeToIso; tc->type = JT_UTF8; } else { PRINTMARK(); - // TODO: last argument here is unused; should decouple string - // from long datetimelike conversion routines - PyDateTimeToJSON(obj, tc, &(GET_TC(tc)->longValue), 0); + NPY_DATETIMEUNIT base = ((PyObjectEncoder *)tc->encoder)->datetimeUnit; + GET_TC(tc)->longValue = PyDateTimeToEpoch(obj, base); tc->type = JT_LONG; } return; @@ -1801,7 +1781,7 @@ void Object_beginTypeContext(JSOBJ _obj, JSONTypeContext *tc) { pc->PyTypeToJSON = PyTimeToJSON; tc->type = JT_UTF8; return; - } else if (PyArray_IsScalar(obj, Datetime)) { + } /* else if (PyArray_IsScalar(obj, Datetime)) { PRINTMARK(); if (((PyDatetimeScalarObject *)obj)->obval == get_nat()) { PRINTMARK(); @@ -1813,7 +1793,7 @@ void Object_beginTypeContext(JSOBJ _obj, JSONTypeContext *tc) { pc->PyTypeToJSON = NpyDateTimeScalarToJSON; tc->type = enc->datetimeIso ? JT_UTF8 : JT_LONG; return; - } else if (PyDelta_Check(obj)) { + } */ else if (PyDelta_Check(obj)) { if (PyObject_HasAttrString(obj, "value")) { PRINTMARK(); value = get_long_attr(obj, "value"); From 6b6b669f4f0480d7b3e238c275bf5960bdcb6327 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sun, 15 Dec 2019 10:54:59 -0800 Subject: [PATCH 03/14] Fixed issue with npy_datetime --- pandas/_libs/src/ujson/python/objToJSON.c | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/pandas/_libs/src/ujson/python/objToJSON.c b/pandas/_libs/src/ujson/python/objToJSON.c index 4f9dd40623ed5..26066a0b5a97b 100644 --- a/pandas/_libs/src/ujson/python/objToJSON.c +++ b/pandas/_libs/src/ujson/python/objToJSON.c @@ -494,7 +494,7 @@ static npy_datetime PyDateTimeToEpoch(PyObject *obj, NPY_DATETIMEUNIT base) { } // TODO: is setting errMsg required? //((JSONObjectEncoder *)tc->encoder)->errorMsg = ""; - return NULL; + // return NULL; } npy_datetime npy_dt = npy_datetimestruct_to_datetime(base, &dts); @@ -1746,15 +1746,30 @@ void Object_beginTypeContext(JSOBJ _obj, JSONTypeContext *tc) { return; } else if (PyTypeNum_ISDATETIME(enc->npyType)) { PRINTMARK(); + if (enc->datetimeIso) { PRINTMARK(); pc->PyTypeToJSON = NpyDateTimeToIso; tc->type = JT_UTF8; } else { PRINTMARK(); - NPY_DATETIMEUNIT base = ((PyObjectEncoder *)tc->encoder)->datetimeUnit; - GET_TC(tc)->longValue = NpyDateTimeToEpoch(obj, base); - tc->type = JT_LONG; + int64_t longVal; + PyArray_VectorUnaryFunc *castfunc = + PyArray_GetCastFunc(PyArray_DescrFromType(enc->npyType), NPY_INT64); + if (!castfunc) { + PyErr_Format(PyExc_ValueError, "Cannot cast numpy dtype %d to long", + enc->npyType); + } + castfunc(enc->npyValue, &longVal, 1, NULL, NULL); + + if (longVal == get_nat()) { + PRINTMARK(); + tc->type = JT_NULL; + } else { + NPY_DATETIMEUNIT base = ((PyObjectEncoder *)tc->encoder)->datetimeUnit; + GET_TC(tc)->longValue = NpyDateTimeToEpoch(longVal, base); + tc->type = JT_LONG; + } } return; } else if (PyDateTime_Check(obj) || PyDate_Check(obj)) { From ffd07a16e293ec40ebfe75598ebaa8a513009cc2 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sun, 15 Dec 2019 10:58:08 -0800 Subject: [PATCH 04/14] Fixed ISO str len return --- pandas/_libs/src/ujson/python/objToJSON.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pandas/_libs/src/ujson/python/objToJSON.c b/pandas/_libs/src/ujson/python/objToJSON.c index 26066a0b5a97b..1d5504c73d0ab 100644 --- a/pandas/_libs/src/ujson/python/objToJSON.c +++ b/pandas/_libs/src/ujson/python/objToJSON.c @@ -434,6 +434,9 @@ static char *NpyDateTimeToIso(npy_datetime dt, JSONTypeContext *tc, void *unused PyObject_Free(result); } + // Note that get_datetime_iso_8601_strlen just gives a generic size + // for ISO string conversion, not the actual size used + *len = strlen(result); return result; } @@ -475,6 +478,9 @@ static char *PyDateTimeToIso(PyObject *obj, JSONTypeContext *tc, void *unused, return NULL; } + // Note that get_datetime_iso_8601_strlen just gives a generic size + // for ISO string conversion, not the actual size used + *len = strlen(result); return result; } From 9f7d9ab15164bc3cd887bcecf3212e35c1322d46 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sun, 15 Dec 2019 11:23:49 -0800 Subject: [PATCH 05/14] infinite loop fix --- pandas/_libs/src/ujson/python/objToJSON.c | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/pandas/_libs/src/ujson/python/objToJSON.c b/pandas/_libs/src/ujson/python/objToJSON.c index 1d5504c73d0ab..4fe87a51c0f4e 100644 --- a/pandas/_libs/src/ujson/python/objToJSON.c +++ b/pandas/_libs/src/ujson/python/objToJSON.c @@ -1777,6 +1777,10 @@ void Object_beginTypeContext(JSOBJ _obj, JSONTypeContext *tc) { tc->type = JT_LONG; } } + + // TODO: this prevents infinite loop with mixed-type DataFrames; refactor + enc->npyCtxtPassthru = NULL; + enc->npyType = -1; return; } else if (PyDateTime_Check(obj) || PyDate_Check(obj)) { if (PyObject_TypeCheck(obj, cls_nat)) { @@ -1802,7 +1806,7 @@ void Object_beginTypeContext(JSOBJ _obj, JSONTypeContext *tc) { pc->PyTypeToJSON = PyTimeToJSON; tc->type = JT_UTF8; return; - } /* else if (PyArray_IsScalar(obj, Datetime)) { + } else if (PyArray_IsScalar(obj, Datetime)) { PRINTMARK(); if (((PyDatetimeScalarObject *)obj)->obval == get_nat()) { PRINTMARK(); @@ -1811,10 +1815,18 @@ void Object_beginTypeContext(JSOBJ _obj, JSONTypeContext *tc) { } PRINTMARK(); - pc->PyTypeToJSON = NpyDateTimeScalarToJSON; - tc->type = enc->datetimeIso ? JT_UTF8 : JT_LONG; + if (enc->datetimeIso) { + PRINTMARK(); + pc->PyTypeToJSON = PyDateTimeToIso; + tc->type = JT_UTF8; + } else { + PRINTMARK(); + NPY_DATETIMEUNIT base = ((PyObjectEncoder *)tc->encoder)->datetimeUnit; + GET_TC(tc)->longValue = PyDateTimeToEpoch(obj, base); + tc->type = JT_LONG; + } return; - } */ else if (PyDelta_Check(obj)) { + } else if (PyDelta_Check(obj)) { if (PyObject_HasAttrString(obj, "value")) { PRINTMARK(); value = get_long_attr(obj, "value"); From 3fc9b908b04c4f9103c07479aaa5646a6a2e73ce Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sun, 15 Dec 2019 12:02:36 -0800 Subject: [PATCH 06/14] revert encode labels typo --- pandas/_libs/src/ujson/python/objToJSON.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/_libs/src/ujson/python/objToJSON.c b/pandas/_libs/src/ujson/python/objToJSON.c index 4fe87a51c0f4e..d4d5389b71e19 100644 --- a/pandas/_libs/src/ujson/python/objToJSON.c +++ b/pandas/_libs/src/ujson/python/objToJSON.c @@ -1566,7 +1566,7 @@ char **NpyArr_encodeLabels(PyArrayObject *labels, PyObjectEncoder *enc, cLabel = (char *)PyUnicode_AsUTF8(iso); Py_DECREF(iso); len = strlen(cLabel); - } else if (PyTypeNum_ISDATETIME(enc->npyType) || PyDateTime_Check(item) || + } else if (PyTypeNum_ISDATETIME(type_num) || PyDateTime_Check(item) || PyDate_Check(item)) { PyObject *ts = PyObject_CallFunction(cls_timestamp, "(O)", item); if (ts == NULL) { From 7471721621ca869a3823b69cbc25a969d7e9f5bc Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sun, 15 Dec 2019 14:38:28 -0800 Subject: [PATCH 07/14] dt iso fix --- pandas/_libs/src/ujson/python/objToJSON.c | 75 ++++++++++++----------- 1 file changed, 40 insertions(+), 35 deletions(-) diff --git a/pandas/_libs/src/ujson/python/objToJSON.c b/pandas/_libs/src/ujson/python/objToJSON.c index d4d5389b71e19..2d0c6538a8d7d 100644 --- a/pandas/_libs/src/ujson/python/objToJSON.c +++ b/pandas/_libs/src/ujson/python/objToJSON.c @@ -410,13 +410,15 @@ static void *PyUnicodeToUTF8(JSOBJ _obj, JSONTypeContext *tc, void *outValue, /* returns a char* and mutates the pointer to *len */ -static char *NpyDateTimeToIso(npy_datetime dt, JSONTypeContext *tc, void *unused, +static char *NpyDateTimeToIso(JSOBJ unused, JSONTypeContext *tc, void *_unused, size_t *len) { npy_datetimestruct dts; - NPY_DATETIMEUNIT base = ((PyObjectEncoder *)tc->encoder)->datetimeUnit; int ret_code; + int64_t longVal = GET_TC(tc)->longValue; - pandas_datetime_to_datetimestruct(dt, base, &dts); + pandas_datetime_to_datetimestruct(longVal, NPY_FR_ns, &dts); + + NPY_DATETIMEUNIT base = ((PyObjectEncoder *)tc->encoder)->datetimeUnit; *len = (size_t)get_datetime_iso_8601_strlen(0, base); char *result = PyObject_Malloc(*len); @@ -1706,6 +1708,41 @@ void Object_beginTypeContext(JSOBJ _obj, JSONTypeContext *tc) { } tc->prv = pc; + if (PyTypeNum_ISDATETIME(enc->npyType)) { + PRINTMARK(); + int64_t longVal; + PyArray_VectorUnaryFunc *castfunc = + PyArray_GetCastFunc(PyArray_DescrFromType(enc->npyType), NPY_INT64); + if (!castfunc) { + PyErr_Format(PyExc_ValueError, "Cannot cast numpy dtype %d to long", + enc->npyType); + } + castfunc(enc->npyValue, &longVal, 1, NULL, NULL); + if (longVal == get_nat()) { + PRINTMARK(); + tc->type = JT_NULL; + } else { + + if (enc->datetimeIso) { + PRINTMARK(); + pc->PyTypeToJSON = NpyDateTimeToIso; + // Currently no way to pass longVal to iso function, so use state management + GET_TC(tc)->longValue = longVal; + tc->type = JT_UTF8; + } else { + PRINTMARK(); + NPY_DATETIMEUNIT base = ((PyObjectEncoder *)tc->encoder)->datetimeUnit; + GET_TC(tc)->longValue = NpyDateTimeToEpoch(longVal, base); + tc->type = JT_LONG; + } + } + + // TODO: this prevents infinite loop with mixed-type DataFrames; refactor + enc->npyCtxtPassthru = NULL; + enc->npyType = -1; + return; + } + if (PyIter_Check(obj) || (PyArray_Check(obj) && !PyArray_CheckScalar(obj))) { PRINTMARK(); @@ -1750,38 +1787,6 @@ void Object_beginTypeContext(JSOBJ _obj, JSONTypeContext *tc) { GET_TC(tc)->doubleValue = PyFloat_AsDouble(obj); tc->type = JT_DOUBLE; return; - } else if (PyTypeNum_ISDATETIME(enc->npyType)) { - PRINTMARK(); - - if (enc->datetimeIso) { - PRINTMARK(); - pc->PyTypeToJSON = NpyDateTimeToIso; - tc->type = JT_UTF8; - } else { - PRINTMARK(); - int64_t longVal; - PyArray_VectorUnaryFunc *castfunc = - PyArray_GetCastFunc(PyArray_DescrFromType(enc->npyType), NPY_INT64); - if (!castfunc) { - PyErr_Format(PyExc_ValueError, "Cannot cast numpy dtype %d to long", - enc->npyType); - } - castfunc(enc->npyValue, &longVal, 1, NULL, NULL); - - if (longVal == get_nat()) { - PRINTMARK(); - tc->type = JT_NULL; - } else { - NPY_DATETIMEUNIT base = ((PyObjectEncoder *)tc->encoder)->datetimeUnit; - GET_TC(tc)->longValue = NpyDateTimeToEpoch(longVal, base); - tc->type = JT_LONG; - } - } - - // TODO: this prevents infinite loop with mixed-type DataFrames; refactor - enc->npyCtxtPassthru = NULL; - enc->npyType = -1; - return; } else if (PyDateTime_Check(obj) || PyDate_Check(obj)) { if (PyObject_TypeCheck(obj, cls_nat)) { PRINTMARK(); From a5201f9520df3ee3081f1ee0c92ffb2f46caa6a2 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sun, 15 Dec 2019 14:41:18 -0800 Subject: [PATCH 08/14] Fixed pydatetime truncation --- pandas/_libs/src/ujson/python/objToJSON.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/_libs/src/ujson/python/objToJSON.c b/pandas/_libs/src/ujson/python/objToJSON.c index 2d0c6538a8d7d..12914e8d2beae 100644 --- a/pandas/_libs/src/ujson/python/objToJSON.c +++ b/pandas/_libs/src/ujson/python/objToJSON.c @@ -505,7 +505,7 @@ static npy_datetime PyDateTimeToEpoch(PyObject *obj, NPY_DATETIMEUNIT base) { // return NULL; } - npy_datetime npy_dt = npy_datetimestruct_to_datetime(base, &dts); + npy_datetime npy_dt = npy_datetimestruct_to_datetime(NPY_FR_ns, &dts); return NpyDateTimeToEpoch(npy_dt, base); } From 946633ef739fc7a8fa28f639658baa799fd2b1e7 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sun, 15 Dec 2019 14:41:31 -0800 Subject: [PATCH 09/14] clang format --- pandas/_libs/src/ujson/python/objToJSON.c | 230 +++++++++++----------- 1 file changed, 117 insertions(+), 113 deletions(-) diff --git a/pandas/_libs/src/ujson/python/objToJSON.c b/pandas/_libs/src/ujson/python/objToJSON.c index 12914e8d2beae..5d3d0110bc874 100644 --- a/pandas/_libs/src/ujson/python/objToJSON.c +++ b/pandas/_libs/src/ujson/python/objToJSON.c @@ -405,108 +405,107 @@ static void *PyBytesToUTF8(JSOBJ _obj, JSONTypeContext *tc, void *outValue, static void *PyUnicodeToUTF8(JSOBJ _obj, JSONTypeContext *tc, void *outValue, size_t *_outLen) { - return PyUnicode_AsUTF8AndSize(_obj, _outLen); + return PyUnicode_AsUTF8AndSize(_obj, _outLen); } - /* returns a char* and mutates the pointer to *len */ static char *NpyDateTimeToIso(JSOBJ unused, JSONTypeContext *tc, void *_unused, - size_t *len) { - npy_datetimestruct dts; - int ret_code; - int64_t longVal = GET_TC(tc)->longValue; - - pandas_datetime_to_datetimestruct(longVal, NPY_FR_ns, &dts); - - NPY_DATETIMEUNIT base = ((PyObjectEncoder *)tc->encoder)->datetimeUnit; - *len = (size_t)get_datetime_iso_8601_strlen(0, base); - char *result = PyObject_Malloc(*len); - - if (result == NULL) { - PyErr_NoMemory(); - ((JSONObjectEncoder *)tc->encoder)->errorMsg = ""; - return NULL; - } + size_t *len) { + npy_datetimestruct dts; + int ret_code; + int64_t longVal = GET_TC(tc)->longValue; + + pandas_datetime_to_datetimestruct(longVal, NPY_FR_ns, &dts); + + NPY_DATETIMEUNIT base = ((PyObjectEncoder *)tc->encoder)->datetimeUnit; + *len = (size_t)get_datetime_iso_8601_strlen(0, base); + char *result = PyObject_Malloc(*len); + + if (result == NULL) { + PyErr_NoMemory(); + ((JSONObjectEncoder *)tc->encoder)->errorMsg = ""; + return NULL; + } - ret_code = make_iso_8601_datetime(&dts, result, *len, base); - if (ret_code != 0) { - PyErr_SetString(PyExc_ValueError, - "Could not convert datetime value to string"); - ((JSONObjectEncoder *)tc->encoder)->errorMsg = ""; - PyObject_Free(result); - } + ret_code = make_iso_8601_datetime(&dts, result, *len, base); + if (ret_code != 0) { + PyErr_SetString(PyExc_ValueError, + "Could not convert datetime value to string"); + ((JSONObjectEncoder *)tc->encoder)->errorMsg = ""; + PyObject_Free(result); + } - // Note that get_datetime_iso_8601_strlen just gives a generic size - // for ISO string conversion, not the actual size used - *len = strlen(result); - return result; + // Note that get_datetime_iso_8601_strlen just gives a generic size + // for ISO string conversion, not the actual size used + *len = strlen(result); + return result; } static npy_datetime NpyDateTimeToEpoch(npy_datetime dt, NPY_DATETIMEUNIT base) { scaleNanosecToUnit(&dt, base); return dt; -} +} static char *PyDateTimeToIso(PyObject *obj, JSONTypeContext *tc, void *unused, - size_t *len) { - npy_datetimestruct dts; - int ret; - - if (!PyDateTime_Check(obj)) { - // TODO: raise TypeError - } - - ret = convert_pydatetime_to_datetimestruct(obj, &dts); - if (ret != 0) { - if (!PyErr_Occurred()) { - PyErr_SetString(PyExc_ValueError, - "Could not convert PyDateTime to numpy datetime"); + size_t *len) { + npy_datetimestruct dts; + int ret; + + if (!PyDateTime_Check(obj)) { + // TODO: raise TypeError } - ((JSONObjectEncoder *)tc->encoder)->errorMsg = ""; - return NULL; - } - NPY_DATETIMEUNIT base = ((PyObjectEncoder *)tc->encoder)->datetimeUnit; - *len = (size_t)get_datetime_iso_8601_strlen(0, base); - char *result = PyObject_Malloc(*len); - ret = make_iso_8601_datetime(&dts, result, *len, base); + ret = convert_pydatetime_to_datetimestruct(obj, &dts); + if (ret != 0) { + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ValueError, + "Could not convert PyDateTime to numpy datetime"); + } + ((JSONObjectEncoder *)tc->encoder)->errorMsg = ""; + return NULL; + } - if (ret != 0) { - PRINTMARK(); - PyErr_SetString(PyExc_ValueError, - "Could not convert datetime value to string"); - ((JSONObjectEncoder *)tc->encoder)->errorMsg = ""; - PyObject_Free(result); - return NULL; - } + NPY_DATETIMEUNIT base = ((PyObjectEncoder *)tc->encoder)->datetimeUnit; + *len = (size_t)get_datetime_iso_8601_strlen(0, base); + char *result = PyObject_Malloc(*len); + ret = make_iso_8601_datetime(&dts, result, *len, base); - // Note that get_datetime_iso_8601_strlen just gives a generic size - // for ISO string conversion, not the actual size used - *len = strlen(result); - return result; + if (ret != 0) { + PRINTMARK(); + PyErr_SetString(PyExc_ValueError, + "Could not convert datetime value to string"); + ((JSONObjectEncoder *)tc->encoder)->errorMsg = ""; + PyObject_Free(result); + return NULL; + } + + // Note that get_datetime_iso_8601_strlen just gives a generic size + // for ISO string conversion, not the actual size used + *len = strlen(result); + return result; } static npy_datetime PyDateTimeToEpoch(PyObject *obj, NPY_DATETIMEUNIT base) { - npy_datetimestruct dts; - int ret; - - if (!PyDateTime_Check(obj)) { - // TODO: raise TypeError - } - - ret = convert_pydatetime_to_datetimestruct(obj, &dts); - if (ret != 0) { - if (!PyErr_Occurred()) { - PyErr_SetString(PyExc_ValueError, - "Could not convert PyDateTime to numpy datetime"); + npy_datetimestruct dts; + int ret; + + if (!PyDateTime_Check(obj)) { + // TODO: raise TypeError } - // TODO: is setting errMsg required? - //((JSONObjectEncoder *)tc->encoder)->errorMsg = ""; - // return NULL; - } - npy_datetime npy_dt = npy_datetimestruct_to_datetime(NPY_FR_ns, &dts); - return NpyDateTimeToEpoch(npy_dt, base); + ret = convert_pydatetime_to_datetimestruct(obj, &dts); + if (ret != 0) { + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ValueError, + "Could not convert PyDateTime to numpy datetime"); + } + // TODO: is setting errMsg required? + //((JSONObjectEncoder *)tc->encoder)->errorMsg = ""; + // return NULL; + } + + npy_datetime npy_dt = npy_datetimestruct_to_datetime(NPY_FR_ns, &dts); + return NpyDateTimeToEpoch(npy_dt, base); } static void *PyTimeToJSON(JSOBJ _obj, JSONTypeContext *tc, void *outValue, @@ -1710,38 +1709,41 @@ void Object_beginTypeContext(JSOBJ _obj, JSONTypeContext *tc) { if (PyTypeNum_ISDATETIME(enc->npyType)) { PRINTMARK(); - int64_t longVal; - PyArray_VectorUnaryFunc *castfunc = - PyArray_GetCastFunc(PyArray_DescrFromType(enc->npyType), NPY_INT64); - if (!castfunc) { - PyErr_Format(PyExc_ValueError, "Cannot cast numpy dtype %d to long", - enc->npyType); - } - castfunc(enc->npyValue, &longVal, 1, NULL, NULL); - if (longVal == get_nat()) { - PRINTMARK(); - tc->type = JT_NULL; - } else { - - if (enc->datetimeIso) { - PRINTMARK(); - pc->PyTypeToJSON = NpyDateTimeToIso; - // Currently no way to pass longVal to iso function, so use state management - GET_TC(tc)->longValue = longVal; - tc->type = JT_UTF8; - } else { + int64_t longVal; + PyArray_VectorUnaryFunc *castfunc = + PyArray_GetCastFunc(PyArray_DescrFromType(enc->npyType), NPY_INT64); + if (!castfunc) { + PyErr_Format(PyExc_ValueError, "Cannot cast numpy dtype %d to long", + enc->npyType); + } + castfunc(enc->npyValue, &longVal, 1, NULL, NULL); + if (longVal == get_nat()) { PRINTMARK(); - NPY_DATETIMEUNIT base = ((PyObjectEncoder *)tc->encoder)->datetimeUnit; - GET_TC(tc)->longValue = NpyDateTimeToEpoch(longVal, base); - tc->type = JT_LONG; - } + tc->type = JT_NULL; + } else { + + if (enc->datetimeIso) { + PRINTMARK(); + pc->PyTypeToJSON = NpyDateTimeToIso; + // Currently no way to pass longVal to iso function, so use + // state management + GET_TC(tc)->longValue = longVal; + tc->type = JT_UTF8; + } else { + PRINTMARK(); + NPY_DATETIMEUNIT base = + ((PyObjectEncoder *)tc->encoder)->datetimeUnit; + GET_TC(tc)->longValue = NpyDateTimeToEpoch(longVal, base); + tc->type = JT_LONG; + } } - // TODO: this prevents infinite loop with mixed-type DataFrames; refactor - enc->npyCtxtPassthru = NULL; - enc->npyType = -1; + // TODO: this prevents infinite loop with mixed-type DataFrames; + // refactor + enc->npyCtxtPassthru = NULL; + enc->npyType = -1; return; - } + } if (PyIter_Check(obj) || (PyArray_Check(obj) && !PyArray_CheckScalar(obj))) { @@ -1801,8 +1803,9 @@ void Object_beginTypeContext(JSOBJ _obj, JSONTypeContext *tc) { tc->type = JT_UTF8; } else { PRINTMARK(); - NPY_DATETIMEUNIT base = ((PyObjectEncoder *)tc->encoder)->datetimeUnit; - GET_TC(tc)->longValue = PyDateTimeToEpoch(obj, base); + NPY_DATETIMEUNIT base = + ((PyObjectEncoder *)tc->encoder)->datetimeUnit; + GET_TC(tc)->longValue = PyDateTimeToEpoch(obj, base); tc->type = JT_LONG; } return; @@ -1826,12 +1829,13 @@ void Object_beginTypeContext(JSOBJ _obj, JSONTypeContext *tc) { tc->type = JT_UTF8; } else { PRINTMARK(); - NPY_DATETIMEUNIT base = ((PyObjectEncoder *)tc->encoder)->datetimeUnit; - GET_TC(tc)->longValue = PyDateTimeToEpoch(obj, base); + NPY_DATETIMEUNIT base = + ((PyObjectEncoder *)tc->encoder)->datetimeUnit; + GET_TC(tc)->longValue = PyDateTimeToEpoch(obj, base); tc->type = JT_LONG; } return; - } else if (PyDelta_Check(obj)) { + } else if (PyDelta_Check(obj)) { if (PyObject_HasAttrString(obj, "value")) { PRINTMARK(); value = get_long_attr(obj, "value"); From 087df6712ad579ed2475ddd415427d963c6f6779 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sun, 15 Dec 2019 14:44:38 -0800 Subject: [PATCH 10/14] Renamed JSON to UTF8 --- pandas/_libs/src/ujson/python/objToJSON.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pandas/_libs/src/ujson/python/objToJSON.c b/pandas/_libs/src/ujson/python/objToJSON.c index 5d3d0110bc874..8f4aa5ec2a6a4 100644 --- a/pandas/_libs/src/ujson/python/objToJSON.c +++ b/pandas/_libs/src/ujson/python/objToJSON.c @@ -59,7 +59,7 @@ PyObject *cls_timedelta; npy_int64 get_nat(void) { return NPY_MIN_INT64; } -typedef void *(*PFN_PyTypeToJSON)(JSOBJ obj, JSONTypeContext *ti, +typedef char *(*PFN_PyTypeToUTF8)(JSOBJ obj, JSONTypeContext *ti, void *outValue, size_t *_outLen); typedef struct __NpyArrContext { @@ -94,7 +94,7 @@ typedef struct __TypeContext { JSPFN_ITERNEXT iterNext; JSPFN_ITERGETNAME iterGetName; JSPFN_ITERGETVALUE iterGetValue; - PFN_PyTypeToJSON PyTypeToJSON; + PFN_PyTypeToUTF8 PyTypeToUTF8; PyObject *newObj; PyObject *dictObj; Py_ssize_t index; @@ -1724,7 +1724,7 @@ void Object_beginTypeContext(JSOBJ _obj, JSONTypeContext *tc) { if (enc->datetimeIso) { PRINTMARK(); - pc->PyTypeToJSON = NpyDateTimeToIso; + pc->PyTypeToUTF8 = NpyDateTimeToIso; // Currently no way to pass longVal to iso function, so use // state management GET_TC(tc)->longValue = longVal; @@ -1776,12 +1776,12 @@ void Object_beginTypeContext(JSOBJ _obj, JSONTypeContext *tc) { return; } else if (PyBytes_Check(obj)) { PRINTMARK(); - pc->PyTypeToJSON = PyBytesToUTF8; + pc->PyTypeToUTF8 = PyBytesToUTF8; tc->type = JT_UTF8; return; } else if (PyUnicode_Check(obj)) { PRINTMARK(); - pc->PyTypeToJSON = PyUnicodeToUTF8; + pc->PyTypeToUTF8 = PyUnicodeToUTF8; tc->type = JT_UTF8; return; } else if (PyObject_TypeCheck(obj, type_decimal)) { @@ -1799,7 +1799,7 @@ void Object_beginTypeContext(JSOBJ _obj, JSONTypeContext *tc) { PRINTMARK(); if (enc->datetimeIso) { PRINTMARK(); - pc->PyTypeToJSON = PyDateTimeToIso; + pc->PyTypeToUTF8 = PyDateTimeToIso; tc->type = JT_UTF8; } else { PRINTMARK(); @@ -1811,7 +1811,7 @@ void Object_beginTypeContext(JSOBJ _obj, JSONTypeContext *tc) { return; } else if (PyTime_Check(obj)) { PRINTMARK(); - pc->PyTypeToJSON = PyTimeToJSON; + pc->PyTypeToUTF8 = PyTimeToJSON; tc->type = JT_UTF8; return; } else if (PyArray_IsScalar(obj, Datetime)) { @@ -1825,7 +1825,7 @@ void Object_beginTypeContext(JSOBJ _obj, JSONTypeContext *tc) { PRINTMARK(); if (enc->datetimeIso) { PRINTMARK(); - pc->PyTypeToJSON = PyDateTimeToIso; + pc->PyTypeToUTF8 = PyDateTimeToIso; tc->type = JT_UTF8; } else { PRINTMARK(); @@ -2235,7 +2235,7 @@ void Object_endTypeContext(JSOBJ obj, JSONTypeContext *tc) { const char *Object_getStringValue(JSOBJ obj, JSONTypeContext *tc, size_t *_outLen) { - return GET_TC(tc)->PyTypeToJSON(obj, tc, NULL, _outLen); + return GET_TC(tc)->PyTypeToUTF8(obj, tc, NULL, _outLen); } JSINT64 Object_getLongValue(JSOBJ obj, JSONTypeContext *tc) { From 47c6ea14af1be703cbbe8b1058a2750a19993d64 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sun, 15 Dec 2019 15:04:29 -0800 Subject: [PATCH 11/14] Char qualifiers --- pandas/_libs/src/ujson/python/objToJSON.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pandas/_libs/src/ujson/python/objToJSON.c b/pandas/_libs/src/ujson/python/objToJSON.c index 8f4aa5ec2a6a4..e342a45a889ea 100644 --- a/pandas/_libs/src/ujson/python/objToJSON.c +++ b/pandas/_libs/src/ujson/python/objToJSON.c @@ -396,16 +396,16 @@ static PyObject *get_item(PyObject *obj, Py_ssize_t i) { return ret; } -static void *PyBytesToUTF8(JSOBJ _obj, JSONTypeContext *tc, void *outValue, +static char *PyBytesToUTF8(JSOBJ _obj, JSONTypeContext *tc, void *outValue, size_t *_outLen) { PyObject *obj = (PyObject *)_obj; *_outLen = PyBytes_GET_SIZE(obj); return PyBytes_AS_STRING(obj); } -static void *PyUnicodeToUTF8(JSOBJ _obj, JSONTypeContext *tc, void *outValue, +static char *PyUnicodeToUTF8(JSOBJ _obj, JSONTypeContext *tc, void *outValue, size_t *_outLen) { - return PyUnicode_AsUTF8AndSize(_obj, _outLen); + return (char *)PyUnicode_AsUTF8AndSize(_obj, _outLen); } /* returns a char* and mutates the pointer to *len */ @@ -446,7 +446,7 @@ static npy_datetime NpyDateTimeToEpoch(npy_datetime dt, NPY_DATETIMEUNIT base) { return dt; } -static char *PyDateTimeToIso(PyObject *obj, JSONTypeContext *tc, void *unused, +static char *PyDateTimeToIso(JSOBJ obj, JSONTypeContext *tc, void *unused, size_t *len) { npy_datetimestruct dts; int ret; @@ -508,7 +508,7 @@ static npy_datetime PyDateTimeToEpoch(PyObject *obj, NPY_DATETIMEUNIT base) { return NpyDateTimeToEpoch(npy_dt, base); } -static void *PyTimeToJSON(JSOBJ _obj, JSONTypeContext *tc, void *outValue, +static char *PyTimeToJSON(JSOBJ _obj, JSONTypeContext *tc, void *outValue, size_t *outLen) { PyObject *obj = (PyObject *)_obj; PyObject *str; @@ -533,7 +533,7 @@ static void *PyTimeToJSON(JSOBJ _obj, JSONTypeContext *tc, void *outValue, GET_TC(tc)->newObj = str; *outLen = PyBytes_GET_SIZE(str); - outValue = (void *)PyBytes_AS_STRING(str); + outValue = PyBytes_AS_STRING(str); return outValue; } From 04f2d16146a4b9d0bc6a52001809e0cb1388ea14 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sun, 15 Dec 2019 15:07:41 -0800 Subject: [PATCH 12/14] Removed unused part of signature --- pandas/_libs/src/ujson/python/objToJSON.c | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/pandas/_libs/src/ujson/python/objToJSON.c b/pandas/_libs/src/ujson/python/objToJSON.c index e342a45a889ea..8d971be211451 100644 --- a/pandas/_libs/src/ujson/python/objToJSON.c +++ b/pandas/_libs/src/ujson/python/objToJSON.c @@ -59,8 +59,7 @@ PyObject *cls_timedelta; npy_int64 get_nat(void) { return NPY_MIN_INT64; } -typedef char *(*PFN_PyTypeToUTF8)(JSOBJ obj, JSONTypeContext *ti, - void *outValue, size_t *_outLen); +typedef char *(*PFN_PyTypeToUTF8)(JSOBJ obj, JSONTypeContext *ti, size_t *_outLen); typedef struct __NpyArrContext { PyObject *array; @@ -396,21 +395,18 @@ static PyObject *get_item(PyObject *obj, Py_ssize_t i) { return ret; } -static char *PyBytesToUTF8(JSOBJ _obj, JSONTypeContext *tc, void *outValue, - size_t *_outLen) { +static char *PyBytesToUTF8(JSOBJ _obj, JSONTypeContext *tc, size_t *_outLen) { PyObject *obj = (PyObject *)_obj; *_outLen = PyBytes_GET_SIZE(obj); return PyBytes_AS_STRING(obj); } -static char *PyUnicodeToUTF8(JSOBJ _obj, JSONTypeContext *tc, void *outValue, - size_t *_outLen) { +static char *PyUnicodeToUTF8(JSOBJ _obj, JSONTypeContext *tc, size_t *_outLen) { return (char *)PyUnicode_AsUTF8AndSize(_obj, _outLen); } /* returns a char* and mutates the pointer to *len */ -static char *NpyDateTimeToIso(JSOBJ unused, JSONTypeContext *tc, void *_unused, - size_t *len) { +static char *NpyDateTimeToIso(JSOBJ unused, JSONTypeContext *tc, size_t *len) { npy_datetimestruct dts; int ret_code; int64_t longVal = GET_TC(tc)->longValue; @@ -446,8 +442,7 @@ static npy_datetime NpyDateTimeToEpoch(npy_datetime dt, NPY_DATETIMEUNIT base) { return dt; } -static char *PyDateTimeToIso(JSOBJ obj, JSONTypeContext *tc, void *unused, - size_t *len) { +static char *PyDateTimeToIso(JSOBJ obj, JSONTypeContext *tc, size_t *len) { npy_datetimestruct dts; int ret; @@ -508,8 +503,7 @@ static npy_datetime PyDateTimeToEpoch(PyObject *obj, NPY_DATETIMEUNIT base) { return NpyDateTimeToEpoch(npy_dt, base); } -static char *PyTimeToJSON(JSOBJ _obj, JSONTypeContext *tc, void *outValue, - size_t *outLen) { +static char *PyTimeToJSON(JSOBJ _obj, JSONTypeContext *tc, size_t *outLen) { PyObject *obj = (PyObject *)_obj; PyObject *str; PyObject *tmp; @@ -533,7 +527,7 @@ static char *PyTimeToJSON(JSOBJ _obj, JSONTypeContext *tc, void *outValue, GET_TC(tc)->newObj = str; *outLen = PyBytes_GET_SIZE(str); - outValue = PyBytes_AS_STRING(str); + char *outValue = PyBytes_AS_STRING(str); return outValue; } @@ -2235,7 +2229,7 @@ void Object_endTypeContext(JSOBJ obj, JSONTypeContext *tc) { const char *Object_getStringValue(JSOBJ obj, JSONTypeContext *tc, size_t *_outLen) { - return GET_TC(tc)->PyTypeToUTF8(obj, tc, NULL, _outLen); + return GET_TC(tc)->PyTypeToUTF8(obj, tc, _outLen); } JSINT64 Object_getLongValue(JSOBJ obj, JSONTypeContext *tc) { From f8ed75d0efb9ca66fd3139caa3700e072b82edbd Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sun, 15 Dec 2019 15:10:25 -0800 Subject: [PATCH 13/14] clang format --- pandas/_libs/src/ujson/python/objToJSON.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pandas/_libs/src/ujson/python/objToJSON.c b/pandas/_libs/src/ujson/python/objToJSON.c index 8d971be211451..ae586ed9bce0c 100644 --- a/pandas/_libs/src/ujson/python/objToJSON.c +++ b/pandas/_libs/src/ujson/python/objToJSON.c @@ -59,7 +59,8 @@ PyObject *cls_timedelta; npy_int64 get_nat(void) { return NPY_MIN_INT64; } -typedef char *(*PFN_PyTypeToUTF8)(JSOBJ obj, JSONTypeContext *ti, size_t *_outLen); +typedef char *(*PFN_PyTypeToUTF8)(JSOBJ obj, JSONTypeContext *ti, + size_t *_outLen); typedef struct __NpyArrContext { PyObject *array; @@ -402,7 +403,7 @@ static char *PyBytesToUTF8(JSOBJ _obj, JSONTypeContext *tc, size_t *_outLen) { } static char *PyUnicodeToUTF8(JSOBJ _obj, JSONTypeContext *tc, size_t *_outLen) { - return (char *)PyUnicode_AsUTF8AndSize(_obj, _outLen); + return (char *)PyUnicode_AsUTF8AndSize(_obj, _outLen); } /* returns a char* and mutates the pointer to *len */ From 98cb43267eec34203e2a169cbf7c79bed7e9a9c3 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Mon, 16 Dec 2019 11:19:41 -0800 Subject: [PATCH 14/14] Warnings fixup --- pandas/_libs/src/ujson/python/objToJSON.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pandas/_libs/src/ujson/python/objToJSON.c b/pandas/_libs/src/ujson/python/objToJSON.c index ae586ed9bce0c..9abcb8333b435 100644 --- a/pandas/_libs/src/ujson/python/objToJSON.c +++ b/pandas/_libs/src/ujson/python/objToJSON.c @@ -403,7 +403,7 @@ static char *PyBytesToUTF8(JSOBJ _obj, JSONTypeContext *tc, size_t *_outLen) { } static char *PyUnicodeToUTF8(JSOBJ _obj, JSONTypeContext *tc, size_t *_outLen) { - return (char *)PyUnicode_AsUTF8AndSize(_obj, _outLen); + return (char *)PyUnicode_AsUTF8AndSize(_obj, (Py_ssize_t *)_outLen); } /* returns a char* and mutates the pointer to *len */ @@ -488,8 +488,9 @@ static npy_datetime PyDateTimeToEpoch(PyObject *obj, NPY_DATETIMEUNIT base) { if (!PyDateTime_Check(obj)) { // TODO: raise TypeError } + PyDateTime_Date *dt = (PyDateTime_Date *)obj; - ret = convert_pydatetime_to_datetimestruct(obj, &dts); + ret = convert_pydatetime_to_datetimestruct(dt, &dts); if (ret != 0) { if (!PyErr_Occurred()) { PyErr_SetString(PyExc_ValueError,