Skip to content

Reduce Loosely Typed Functions in objToJSON #29954

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Dec 7, 2019
Merged

Reduce Loosely Typed Functions in objToJSON #29954

Changes from 6 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
6c2416d
Consistent double state management
WillAyd Dec 1, 2019
9710050
Remove CLong
WillAyd Dec 2, 2019
6148878
partial fix for long removal
WillAyd Dec 2, 2019
377c01c
removed comments and unused func
WillAyd Dec 2, 2019
d98bd0e
fixed issue with integer date_unit handling
WillAyd Dec 2, 2019
86a575f
ran clang-format
WillAyd Dec 2, 2019
4bd6d2f
Merge remote-tracking branch 'upstream/master' into consistent-json
WillAyd Dec 2, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
99 changes: 38 additions & 61 deletions pandas/_libs/src/ujson/python/objToJSON.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -385,42 +385,8 @@ static PyObject *get_item(PyObject *obj, Py_ssize_t i) {
return ret;
}

static void *CDouble(JSOBJ obj, JSONTypeContext *tc, void *outValue,
size_t *_outLen) {
PRINTMARK();
*((double *)outValue) = GET_TC(tc)->doubleValue;
return NULL;
}

static void *CLong(JSOBJ obj, JSONTypeContext *tc, void *outValue,
size_t *_outLen) {
PRINTMARK();
*((JSINT64 *)outValue) = GET_TC(tc)->longValue;
return NULL;
}

static void *PyLongToINT64(JSOBJ _obj, JSONTypeContext *tc, void *outValue,
size_t *_outLen) {
*((JSINT64 *)outValue) = GET_TC(tc)->longValue;
return NULL;
}

static void *NpyFloatToDOUBLE(JSOBJ _obj, JSONTypeContext *tc, void *outValue,
size_t *_outLen) {
PyObject *obj = (PyObject *)_obj;
PyArray_CastScalarToCtype(obj, outValue, PyArray_DescrFromType(NPY_DOUBLE));
return NULL;
}

static void *PyFloatToDOUBLE(JSOBJ _obj, JSONTypeContext *tc, void *outValue,
size_t *_outLen) {
PyObject *obj = (PyObject *)_obj;
*((double *)outValue) = PyFloat_AsDouble(obj);
return NULL;
}

static void *PyBytesToUTF8(JSOBJ _obj, JSONTypeContext *tc, void *outValue,
size_t *_outLen) {
size_t *_outLen) {
PyObject *obj = (PyObject *)_obj;
*_outLen = PyBytes_GET_SIZE(obj);
return PyBytes_AS_STRING(obj);
Expand Down Expand Up @@ -571,7 +537,6 @@ static int NpyTypeToJSONType(PyObject *obj, JSONTypeContext *tc, int npyType,
return JT_NULL;
}
GET_TC(tc)->doubleValue = (double)doubleVal;
GET_TC(tc)->PyTypeToJSON = CDouble;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

To make sure i understatnd, Foo._PyTypeToJSON are the callbacks you're referring to?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yep that's right - anything getting assigned to PyTypeToJSON (side note - not technically callbacks but void pointer functions; edited appropriately)

In the end we might be able to get rid of the custom PyEncoder altogether and just use more built-in ujson functionality by assigning the native types and returning JT_<TYPE> where appropriate

return JT_DOUBLE;
}

Expand All @@ -588,10 +553,32 @@ static int NpyTypeToJSONType(PyObject *obj, JSONTypeContext *tc, int npyType,
PRINTMARK();
return JT_NULL;
}
GET_TC(tc)->longValue = (JSINT64)longVal;
GET_TC(tc)->PyTypeToJSON = NpyDatetime64ToJSON;
return ((PyObjectEncoder *)tc->encoder)->datetimeIso ? JT_UTF8
: JT_LONG;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

im not so familiar with this syntax. is the ?:; here equivalent to the if/else you've changed this to?

There is still GET_TC(tc)->PyTypeToJSON = NpyDatetime64ToJSON on 559. You're not getting rid of this one?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That is the ternary operator, so basically <EXPRESSION> ? <VALUE_IF_TRUE> : <VALUE_IF_FALSE>

This is used elsewhere in the module; just copying over

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i think i get it, thanks


if (((PyObjectEncoder *)tc->encoder)->datetimeIso) {
GET_TC(tc)->longValue = (JSINT64)longVal;
GET_TC(tc)->PyTypeToJSON = NpyDatetime64ToJSON;
return JT_UTF8;
} else {

// TODO: consolidate uses of this switch
switch (((PyObjectEncoder *)tc->encoder)->datetimeUnit) {
case NPY_FR_ns:
break;
case NPY_FR_us:
longVal /= 1000LL;
break;
case NPY_FR_ms:
longVal /= 1000000LL;
break;
case NPY_FR_s:
longVal /= 1000000000LL;
break;
default:
break; // TODO: should raise error
}
GET_TC(tc)->longValue = longVal;
return JT_LONG;
}
}

if (PyTypeNum_ISINTEGER(npyType)) {
Expand All @@ -604,7 +591,6 @@ static int NpyTypeToJSONType(PyObject *obj, JSONTypeContext *tc, int npyType,
}
castfunc(value, &longVal, 1, NULL, NULL);
GET_TC(tc)->longValue = (JSINT64)longVal;
GET_TC(tc)->PyTypeToJSON = CLong;
return JT_LONG;
}

Expand Down Expand Up @@ -1829,7 +1815,6 @@ void Object_beginTypeContext(JSOBJ _obj, JSONTypeContext *tc) {

if (PyLong_Check(obj)) {
PRINTMARK();
pc->PyTypeToJSON = PyLongToINT64;
tc->type = JT_LONG;
GET_TC(tc)->longValue = PyLong_AsLongLong(obj);

Expand All @@ -1847,7 +1832,7 @@ void Object_beginTypeContext(JSOBJ _obj, JSONTypeContext *tc) {
if (npy_isnan(val) || npy_isinf(val)) {
tc->type = JT_NULL;
} else {
pc->PyTypeToJSON = PyFloatToDOUBLE;
GET_TC(tc)->doubleValue = val;
tc->type = JT_DOUBLE;
}
return;
Expand All @@ -1863,7 +1848,7 @@ void Object_beginTypeContext(JSOBJ _obj, JSONTypeContext *tc) {
return;
} else if (PyObject_TypeCheck(obj, type_decimal)) {
PRINTMARK();
pc->PyTypeToJSON = PyFloatToDOUBLE;
GET_TC(tc)->doubleValue = PyFloat_AsDouble(obj);
tc->type = JT_DOUBLE;
return;
} else if (PyDateTime_Check(obj) || PyDate_Check(obj)) {
Expand All @@ -1874,12 +1859,15 @@ void Object_beginTypeContext(JSOBJ _obj, JSONTypeContext *tc) {
}

PRINTMARK();
pc->PyTypeToJSON = PyDateTimeToJSON;
if (enc->datetimeIso) {
PRINTMARK();
pc->PyTypeToJSON = PyDateTimeToJSON;
tc->type = JT_UTF8;
} else {
PRINTMARK();
// TODO: last argument here is unused; should decouple string
// from long datetimelike conversion routines
PyDateTimeToJSON(obj, tc, &(GET_TC(tc)->longValue), 0);
tc->type = JT_LONG;
}
return;
Expand Down Expand Up @@ -1940,12 +1928,10 @@ void Object_beginTypeContext(JSOBJ _obj, JSONTypeContext *tc) {
GET_TC(tc)->longValue = value;

PRINTMARK();
pc->PyTypeToJSON = PyLongToINT64;
tc->type = JT_LONG;
return;
} else if (PyArray_IsScalar(obj, Integer)) {
PRINTMARK();
pc->PyTypeToJSON = PyLongToINT64;
tc->type = JT_LONG;
PyArray_CastScalarToCtype(obj, &(GET_TC(tc)->longValue),
PyArray_DescrFromType(NPY_INT64));
Expand All @@ -1966,7 +1952,8 @@ void Object_beginTypeContext(JSOBJ _obj, JSONTypeContext *tc) {
return;
} else if (PyArray_IsScalar(obj, Float) || PyArray_IsScalar(obj, Double)) {
PRINTMARK();
pc->PyTypeToJSON = NpyFloatToDOUBLE;
PyArray_CastScalarToCtype(obj, &(GET_TC(tc)->doubleValue),
PyArray_DescrFromType(NPY_DOUBLE));
tc->type = JT_DOUBLE;
return;
} else if (PyArray_Check(obj) && PyArray_CheckScalar(obj)) {
Expand Down Expand Up @@ -2315,21 +2302,11 @@ const char *Object_getStringValue(JSOBJ obj, JSONTypeContext *tc,
}

JSINT64 Object_getLongValue(JSOBJ obj, JSONTypeContext *tc) {
JSINT64 ret;
GET_TC(tc)->PyTypeToJSON(obj, tc, &ret, NULL);
return ret;
}

JSINT32 Object_getIntValue(JSOBJ obj, JSONTypeContext *tc) {
JSINT32 ret;
GET_TC(tc)->PyTypeToJSON(obj, tc, &ret, NULL);
return ret;
return GET_TC(tc)->longValue;
}

double Object_getDoubleValue(JSOBJ obj, JSONTypeContext *tc) {
double ret;
GET_TC(tc)->PyTypeToJSON(obj, tc, &ret, NULL);
return ret;
return GET_TC(tc)->doubleValue;
}

static void Object_releaseObject(JSOBJ _obj) { Py_DECREF((PyObject *)_obj); }
Expand Down Expand Up @@ -2384,7 +2361,7 @@ PyObject *objToJSON(PyObject *self, PyObject *args, PyObject *kwargs) {
Object_endTypeContext,
Object_getStringValue,
Object_getLongValue,
Object_getIntValue,
NULL, // getIntValue is unused
Object_getDoubleValue,
Object_iterBegin,
Object_iterNext,
Expand Down