From afbdfdb3ae9afabefe64bd3a6662ba7a6dbf406e Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 13 Dec 2018 16:50:36 +0900 Subject: [PATCH 1/3] charsetnr should not be used for FIELD_TYPE_JSON --- MySQLdb/_mysql.c | 105 +++++++++++++++++++++-------------------- MySQLdb/connections.py | 5 +- 2 files changed, 57 insertions(+), 53 deletions(-) diff --git a/MySQLdb/_mysql.c b/MySQLdb/_mysql.c index 08ea9ae0..a29fdb0b 100644 --- a/MySQLdb/_mysql.c +++ b/MySQLdb/_mysql.c @@ -312,10 +312,13 @@ _mysql_ResultObject_Initialize( long flags = fields[i].flags; PyObject *fun2=NULL; int j, n2=PySequence_Size(fun); - if (fields[i].charsetnr != 63) { /* maaagic */ - flags &= ~BINARY_FLAG; - } else { + // BINARY_FLAG means ***_bin collation is used. + // To distinguish text and binary, we shoud use charsetnr==63 (binary). + // But we abuse BINARY_FLAG for historical reason. + if (fields[i].charsetnr == 63) { flags |= BINARY_FLAG; + } else { + flags &= ~BINARY_FLAG; } for (j=0; jtype; - // Return bytes for binary and string types. - int binary = 0; - if (field_type == FIELD_TYPE_TINY_BLOB || - field_type == FIELD_TYPE_MEDIUM_BLOB || - field_type == FIELD_TYPE_LONG_BLOB || - field_type == FIELD_TYPE_BLOB || - field_type == FIELD_TYPE_VAR_STRING || - field_type == FIELD_TYPE_STRING || - field_type == FIELD_TYPE_GEOMETRY || - field_type == FIELD_TYPE_BIT) { - binary = 1; + if (rowitem == NULL) { + Py_RETURN_NONE; } -#endif - if (rowitem) { - if (converter == (PyObject*)&PyUnicode_Type) { - if (encoding == utf8) { - //fprintf(stderr, "decoding with utf8!\n"); - v = PyUnicode_DecodeUTF8(rowitem, length, NULL); - } else { - //fprintf(stderr, "decoding with %s\n", encoding); - v = PyUnicode_Decode(rowitem, length, encoding, NULL); - } - } - else if (converter == (PyObject*)&PyBytes_Type || converter == Py_None) { - //fprintf(stderr, "decoding with bytes\n", encoding); - v = PyBytes_FromStringAndSize(rowitem, length); - } - else if (converter == (PyObject*)&PyInt_Type) { - //fprintf(stderr, "decoding with int\n", encoding); - v = PyInt_FromString(rowitem, NULL, 10); + + // Fast paths for int, string and binary. + if (converter == (PyObject*)&PyUnicode_Type) { + if (encoding == utf8) { + //fprintf(stderr, "decoding with utf8!\n"); + return PyUnicode_DecodeUTF8(rowitem, length, NULL); + } else { + //fprintf(stderr, "decoding with %s\n", encoding); + return PyUnicode_Decode(rowitem, length, encoding, NULL); } - else { - //fprintf(stderr, "decoding with callback\n"); - //PyObject_Print(converter, stderr, 0); - //fprintf(stderr, "\n"); - v = PyObject_CallFunction(converter, + } + if (converter == (PyObject*)&PyBytes_Type || converter == Py_None) { + //fprintf(stderr, "decoding with bytes\n", encoding); + return PyBytes_FromStringAndSize(rowitem, length); + } + if (converter == (PyObject*)&PyInt_Type) { + //fprintf(stderr, "decoding with int\n", encoding); + return PyInt_FromString(rowitem, NULL, 10); + } + + //fprintf(stderr, "decoding with callback\n"); + //PyObject_Print(converter, stderr, 0); + //fprintf(stderr, "\n"); #ifdef IS_PY3K - binary ? "y#" : "s#", -#else - "s#", -#endif - rowitem, - (int)length); - } - } else { - Py_INCREF(Py_None); - v = Py_None; + int binary; + switch (field->type) { + case FIELD_TYPE_TINY_BLOB: + case FIELD_TYPE_MEDIUM_BLOB: + case FIELD_TYPE_LONG_BLOB: + case FIELD_TYPE_BLOB: + case FIELD_TYPE_VAR_STRING: + case FIELD_TYPE_STRING: + case FIELD_TYPE_JSON: + case FIELD_TYPE_GEOMETRY: + case FIELD_TYPE_BIT: + // Call converter with bytes + binary = 1; + default: // e.g. FIELD_TYPE_DATETIME, etc. + // Call converter with unicode string + binary = 0; } - return v; + return PyObject_CallFunction(converter, + binary ? "y#" : "s#", + rowitem, (int)length); +#else + return PyObject_CallFunction(converter, + "s#", rowitem, (int)length); } static PyObject * @@ -1236,7 +1237,7 @@ _mysql_row_to_dict_old( unsigned int n, i; unsigned long *length; PyObject *r, *c; - MYSQL_FIELD *fields; + MYSQL_FIELD *fields; n = mysql_num_fields(self->result); if (!(r = PyDict_New())) return NULL; diff --git a/MySQLdb/connections.py b/MySQLdb/connections.py index 29cc4f32..7de8c8c2 100644 --- a/MySQLdb/connections.py +++ b/MySQLdb/connections.py @@ -183,8 +183,11 @@ def unicode_literal(u, dummy=None): if use_unicode: for t in (FIELD_TYPE.STRING, FIELD_TYPE.VAR_STRING, FIELD_TYPE.VARCHAR, FIELD_TYPE.TINY_BLOB, - FIELD_TYPE.MEDIUM_BLOB, FIELD_TYPE.LONG_BLOB, FIELD_TYPE.BLOB, FIELD_TYPE.JSON): + FIELD_TYPE.MEDIUM_BLOB, FIELD_TYPE.LONG_BLOB, FIELD_TYPE.BLOB): self.converter[t] = _bytes_or_str + # Unlike other string/blob types, JSON is always text. + # MySQL may return JSON with charset==binary. + self.converter[FIELD_TYPE.JSON] = unicode self.encoders[unicode] = unicode_literal self._transactional = self.server_capabilities & CLIENT.TRANSACTIONS From 8f136c504c25712a6ff999ea38319ddae213feff Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 13 Dec 2018 17:10:51 +0900 Subject: [PATCH 2/3] FIELD_TYPE_JSON is not defined --- MySQLdb/_mysql.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/MySQLdb/_mysql.c b/MySQLdb/_mysql.c index a29fdb0b..37243fad 100644 --- a/MySQLdb/_mysql.c +++ b/MySQLdb/_mysql.c @@ -1146,9 +1146,13 @@ _mysql_field_to_python( case FIELD_TYPE_BLOB: case FIELD_TYPE_VAR_STRING: case FIELD_TYPE_STRING: - case FIELD_TYPE_JSON: case FIELD_TYPE_GEOMETRY: case FIELD_TYPE_BIT: +#ifdef FIELD_TYPE_JSON + case FIELD_TYPE_JSON: +#else + case 245: // JSON +#endif // Call converter with bytes binary = 1; default: // e.g. FIELD_TYPE_DATETIME, etc. From 6b66da642958f5927c0b4f0bf525ac627b589c8b Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 13 Dec 2018 17:28:09 +0900 Subject: [PATCH 3/3] Add missing #endif --- MySQLdb/_mysql.c | 1 + 1 file changed, 1 insertion(+) diff --git a/MySQLdb/_mysql.c b/MySQLdb/_mysql.c index 37243fad..2ae8ce8a 100644 --- a/MySQLdb/_mysql.c +++ b/MySQLdb/_mysql.c @@ -1165,6 +1165,7 @@ _mysql_field_to_python( #else return PyObject_CallFunction(converter, "s#", rowitem, (int)length); +#endif } static PyObject *