From 2b95a956ffa48c7430930288053ea1baf9b7a667 Mon Sep 17 00:00:00 2001 From: dongwook-chan Date: Mon, 23 Aug 2021 02:24:51 +0900 Subject: [PATCH 1/3] Fix row events for MySQL8 partitioned table Since MySQL 8.0.16, partition info has been [added](https://github.com/mysql/mysql-server/commit/e11a540fc559dfbed72ba4ec4677638ac917454f) into extra_data in row event. Current code is aware of extra_data but doesn't parse it accurately. Its length is defined as 'extra_data_length / 8'. However, the [maximum length of partition info](https://github.com/mysql/mysql-server/blob/beb865a960b9a8a16cf999c323e46c5b0c67f21f/libbinlogevents/include/rows_event.h#L772-L814) is 5. Wrong definition leads program to read 0 (= 5/8) bytes of extra_data. This causes any row event on partitioned table leads to packet parse failure (undefined behavior). In some cases errors occur, and in other cases parse result shows incorrect filed values as in [issue #354](https://github.com/noplay/python-mysql-replication/issues/354#issue-971136865). 1. Fix byte-length of extra_data to conform to repl protocol extra_data_length / 8 -> extra_data_length - 2 Refer to [MySQL Document](https://dev.mysql.com/doc/internals/en/rows-event.html) and [MySQL source code]((https://github.com/mysql/mysql-server/blob/beb865a960b9a8a16cf999c323e46c5b0c67f21f/libbinlogevents/src/rows_event.cpp#L415)). Above fixes failure of packet parsing, but following modification is further required to get partition info. 2. Parse extra_data according to [partition info layout](https://github.com/mysql/mysql-server/blob/beb865a960b9a8a16cf999c323e46c5b0c67f21f/libbinlogevents/include/rows_event.h#L772-L814). --- pymysqlreplication/row_event.py | 12 +++++++++++- pymysqlreplication/tests/base.py | 4 ++++ pymysqlreplication/tests/test_data_type.py | 16 ++++++++++++++++ 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/pymysqlreplication/row_event.py b/pymysqlreplication/row_event.py index 0e1c3feb..7497e315 100644 --- a/pymysqlreplication/row_event.py +++ b/pymysqlreplication/row_event.py @@ -57,7 +57,17 @@ def __init__(self, from_packet, event_size, table_map, ctl_connection, **kwargs) self.event_type == BINLOG.DELETE_ROWS_EVENT_V2 or \ self.event_type == BINLOG.UPDATE_ROWS_EVENT_V2: self.flags, self.extra_data_length = struct.unpack(' 2: + self.extra_data_type = struct.unpack('= 8.0 + @property def supportsGTID(self): if not self.isMySQL56AndMore(): diff --git a/pymysqlreplication/tests/test_data_type.py b/pymysqlreplication/tests/test_data_type.py index 6c4afed1..3a3fdb7d 100644 --- a/pymysqlreplication/tests/test_data_type.py +++ b/pymysqlreplication/tests/test_data_type.py @@ -625,5 +625,21 @@ def test_zerofill(self): self.assertEqual(event.rows[0]["values"]["test4"], '0000000001') self.assertEqual(event.rows[0]["values"]["test5"], '00000000000000000001') + def test_extra_data(self): + if not self.isMySQL80AndMore(): + self.skipTest("Not supported in this version of MySQL") + create_query = "CREATE TABLE test (id INTEGER) \ + PARTITION BY RANGE (id) ( \ + PARTITION p0 VALUES LESS THAN (1), \ + PARTITION p1 VALUES LESS THAN (2), \ + PARTITION p2 VALUES LESS THAN (3), \ + PARTITION p3 VALUES LESS THAN (4), \ + PARTITION p4 VALUES LESS THAN (5), \ + )" + insert_query = "INSERT INTO test (id) VALUES(3)" + event = self.create_and_insert_value(create_query, insert_query) + self.assertEqual(event.extra_data_type, 1) + self.assertEqual(event.partition_id, 3) + if __name__ == "__main__": unittest.main() From 6c363332a938a8d3d2795581ce316c2f794f7071 Mon Sep 17 00:00:00 2001 From: dongwook-chan Date: Mon, 23 Aug 2021 23:21:54 +0900 Subject: [PATCH 2/3] Support parsing of ndb info Refer to [ndb info layout](https://github.com/mysql/mysql-server/blob/beb865a960b9a8a16cf999c323e46c5b0c67f21f/libbinlogevents/include/rows_event.h#L782-L788) --- pymysqlreplication/row_event.py | 11 ++++++++--- pymysqlreplication/tests/test_data_type.py | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/pymysqlreplication/row_event.py b/pymysqlreplication/row_event.py index 7497e315..832c6688 100644 --- a/pymysqlreplication/row_event.py +++ b/pymysqlreplication/row_event.py @@ -59,15 +59,20 @@ def __init__(self, from_packet, event_size, table_map, ctl_connection, **kwargs) self.flags, self.extra_data_length = struct.unpack(' 2: self.extra_data_type = struct.unpack(' Date: Mon, 23 Aug 2021 23:30:15 +0900 Subject: [PATCH 3/3] Change test name test_extra_data -> test_partition_id --- pymysqlreplication/tests/test_data_type.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymysqlreplication/tests/test_data_type.py b/pymysqlreplication/tests/test_data_type.py index 67866928..e94aa6ad 100644 --- a/pymysqlreplication/tests/test_data_type.py +++ b/pymysqlreplication/tests/test_data_type.py @@ -625,7 +625,7 @@ def test_zerofill(self): self.assertEqual(event.rows[0]["values"]["test4"], '0000000001') self.assertEqual(event.rows[0]["values"]["test5"], '00000000000000000001') - def test_extra_data(self): + def test_partition_id(self): if not self.isMySQL80AndMore(): self.skipTest("Not supported in this version of MySQL") create_query = "CREATE TABLE test (id INTEGER) \