From eae3c260ac49ec62ecdaaa2963b7c963cf0198ca Mon Sep 17 00:00:00 2001 From: howmp Date: Tue, 10 Jul 2018 20:36:55 +0800 Subject: [PATCH 1/2] Fix JSON field to decode bytes in python3 --- pymysqlreplication/packet.py | 4 ++-- pymysqlreplication/tests/test_data_type.py | 23 ++++++---------------- 2 files changed, 8 insertions(+), 19 deletions(-) diff --git a/pymysqlreplication/packet.py b/pymysqlreplication/packet.py index 936dc8c5..6ccf4274 100644 --- a/pymysqlreplication/packet.py +++ b/pymysqlreplication/packet.py @@ -266,7 +266,7 @@ def read_variable_length_string(self): byte = byte2int(self.read(1)) length = length | ((byte & 0x7f) << bits_read) bits_read = bits_read + 7 - return self.read(length) + return self.read(length).decode() def read_int24(self): a, b, c = struct.unpack("BBB", self.read(3)) @@ -429,7 +429,7 @@ def read_binary_json_object(self, length, large): value_type_inlined_lengths = [read_offset_or_inline(self, large) for _ in range(elements)] - keys = [self.read(x[1]) for x in key_offset_lengths] + keys = [self.read(x[1]).decode() for x in key_offset_lengths] out = {} for i in range(elements): diff --git a/pymysqlreplication/tests/test_data_type.py b/pymysqlreplication/tests/test_data_type.py index 4e799db9..a9b22106 100644 --- a/pymysqlreplication/tests/test_data_type.py +++ b/pymysqlreplication/tests/test_data_type.py @@ -19,17 +19,6 @@ __all__ = ["TestDataType"] - -def to_binary_dict(d): - def encode_value(v): - if isinstance(v, text_type): - return v.encode() - if isinstance(v, list): - return [encode_value(x) for x in v] - return v - return dict([(k.encode(), encode_value(v)) for (k, v) in d.items()]) - - class TestDataType(base.PyMySQLReplicationTestCase): def ignoredEvents(self): return [GtidEvent] @@ -431,7 +420,7 @@ def test_json(self): create_query = "CREATE TABLE test (id int, value json);" insert_query = """INSERT INTO test (id, value) VALUES (1, '{"my_key": "my_val", "my_key2": "my_val2"}');""" event = self.create_and_insert_value(create_query, insert_query) - self.assertEqual(event.rows[0]["values"]["value"], {b"my_key": b"my_val", b"my_key2": b"my_val2"}) + self.assertEqual(event.rows[0]["values"]["value"], {"my_key": "my_val", "my_key2": "my_val2"}) def test_json_array(self): if not self.isMySQL57(): @@ -439,7 +428,7 @@ def test_json_array(self): create_query = "CREATE TABLE test (id int, value json);" insert_query = """INSERT INTO test (id, value) VALUES (1, '["my_val", "my_val2"]');""" event = self.create_and_insert_value(create_query, insert_query) - self.assertEqual(event.rows[0]["values"]["value"], [b'my_val', b'my_val2']) + self.assertEqual(event.rows[0]["values"]["value"], ['my_val', 'my_val2']) def test_json_large(self): if not self.isMySQL57(): @@ -449,7 +438,7 @@ def test_json_large(self): insert_query = """INSERT INTO test (id, value) VALUES (1, '%s');""" % json.dumps(data) event = self.create_and_insert_value(create_query, insert_query) - self.assertEqual(event.rows[0]["values"]["value"], to_binary_dict(data)) + self.assertEqual(event.rows[0]["values"]["value"], data) def test_json_types(self): if not self.isMySQL57(): @@ -474,7 +463,7 @@ def test_json_types(self): create_query = "CREATE TABLE test (id int, value json);" insert_query = """INSERT INTO test (id, value) VALUES (1, '%s');""" % json.dumps(data) event = self.create_and_insert_value(create_query, insert_query) - self.assertEqual(event.rows[0]["values"]["value"], to_binary_dict(data)) + self.assertEqual(event.rows[0]["values"]["value"], data) self.tearDown() self.setUp() @@ -511,7 +500,7 @@ def test_json_unicode(self): create_query = "CREATE TABLE test (id int, value json);" insert_query = u"""INSERT INTO test (id, value) VALUES (1, '{"miam": "🍔"}');""" event = self.create_and_insert_value(create_query, insert_query) - self.assertEqual(event.rows[0]["values"]["value"][b"miam"], u'🍔'.encode('utf8')) + self.assertEqual(event.rows[0]["values"]["value"]["miam"], u'🍔') def test_json_long_string(self): if not self.isMySQL57(): @@ -521,7 +510,7 @@ def test_json_long_string(self): string_value = "super_long_string" * 100 insert_query = "INSERT INTO test (id, value) VALUES (1, '{\"my_key\": \"%s\"}');" % (string_value,) event = self.create_and_insert_value(create_query, insert_query) - self.assertEqual(event.rows[0]["values"]["value"], to_binary_dict({"my_key": string_value})) + self.assertEqual(event.rows[0]["values"]["value"], {"my_key": string_value}) def test_null(self): create_query = "CREATE TABLE test ( \ From fd824704684d9d7cb2b28690a9eadaa36e5a01e9 Mon Sep 17 00:00:00 2001 From: howmp Date: Tue, 10 Jul 2018 20:44:40 +0800 Subject: [PATCH 2/2] fix decode in python2 --- pymysqlreplication/packet.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pymysqlreplication/packet.py b/pymysqlreplication/packet.py index 6ccf4274..dd38f729 100644 --- a/pymysqlreplication/packet.py +++ b/pymysqlreplication/packet.py @@ -266,7 +266,7 @@ def read_variable_length_string(self): byte = byte2int(self.read(1)) length = length | ((byte & 0x7f) << bits_read) bits_read = bits_read + 7 - return self.read(length).decode() + return self.read(length).decode('utf-8') def read_int24(self): a, b, c = struct.unpack("BBB", self.read(3)) @@ -429,7 +429,7 @@ def read_binary_json_object(self, length, large): value_type_inlined_lengths = [read_offset_or_inline(self, large) for _ in range(elements)] - keys = [self.read(x[1]).decode() for x in key_offset_lengths] + keys = [self.read(x[1]).decode('utf-8') for x in key_offset_lengths] out = {} for i in range(elements):