Enhance Testing with MySQL8 & Update GitHub Actions (#484)

Co-authored-by: Heeseon Cheon <[email protected]>

Author: mjs1995

.github/workflows/pytest.yml

@@ -13,14 +13,14 @@ jobs:
           - {name: 'Pypy 3.9', python: 'pypy-3.9'}
     name: ${{ matrix.name }}
     runs-on: ubuntu-latest
-    timeout-minutes: 2
+    timeout-minutes: 3
 
     steps:
       - name: Check out code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Setup Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python }}
 

docker-compose-test.yml

@@ -37,6 +37,15 @@ services:
     networks:
       - default
 
+  percona-8.0:
+    <<: *mysql
+    image: percona:8.0
+    platform: linux/amd64
+    ports:
+      - "3309:3306"
+    networks:
+      - default
+
   mariadb-10.6:
     <<: *mariadb
     image: mariadb:10.6
@@ -61,6 +70,8 @@ services:
         MYSQL_5_7: percona-5.7
         MYSQL_5_7_CTL: percona-5.7-ctl
         MYSQL_5_7_CTL_PORT: 3306
+        MYSQL_8_0: percona-8.0
+        MYSQL_8_0_PORT: 3306
         MARIADB_10_6: mariadb-10.6
         MARIADB_10_6_PORT: 3306
 
@@ -72,7 +83,7 @@ services:
         
         while :
         do
-          if mysql -h percona-5.7 --user=root --execute "SELECT version();" 2>&1 >/dev/null && mysql -h percona-5.7-ctl --user=root --execute "SELECT version();" 2>&1 >/dev/null; then
+          if mysql -h percona-5.7 --user=root --execute "SELECT version();" 2>&1 >/dev/null && mysql -h percona-5.7-ctl --user=root --execute "SELECT version();" 2>&1 >/dev/null && mysql -h percona-8.0 --user=root --execute "SELECT version();" 2>&1 >/dev/null; then
             break
           fi
           sleep 1
@@ -87,6 +98,7 @@ services:
     depends_on:
       - percona-5.7
       - percona-5.7-ctl
+      - percona-8.0
 
 networks:
   default:

docker-compose.yml

@@ -33,6 +33,12 @@ services:
     ports:
       - "3307:3306"
 
+  percona-8.0:
+    <<: *mysql
+    image: percona:8.0
+    ports:
+      - "3309:3306"
+
   mariadb-10.6:
     <<: *mariadb
     image: mariadb:10.6

pymysqlreplication/tests/base.py

@@ -160,3 +160,31 @@ def bin_log_basename(self):
         bin_log_basename = cursor.fetchone()[0]
         bin_log_basename = bin_log_basename.split("/")[-1]
         return bin_log_basename
+
+
+class PyMySQLReplicationVersion8TestCase(PyMySQLReplicationTestCase):
+    def setUp(self):
+        super().setUp()
+        # default
+        self.database = {
+            "host": os.environ.get("MYSQL_8_0") or "localhost",
+            "user": "root",
+            "passwd": "",
+            "port": int(os.environ.get("MYSQL_8_0_PORT") or 3309),
+            "use_unicode": True,
+            "charset": "utf8",
+            "db": "pymysqlreplication_test",
+        }
+
+        self.conn_control = None
+        db = copy.copy(self.database)
+        db["db"] = None
+        self.connect_conn_control(db)
+        self.execute("DROP DATABASE IF EXISTS pymysqlreplication_test")
+        self.execute("CREATE DATABASE pymysqlreplication_test")
+        db = copy.copy(self.database)
+        self.connect_conn_control(db)
+        self.stream = None
+        self.resetBinLog()
+        self.isMySQL80AndMore()
+        self.__is_mariaDB = None

pymysqlreplication/tests/test_basic.py

@@ -1478,7 +1478,7 @@ def test_query_event_latin1(self):
         assert event.query == r"CREATE TABLE test_latin1_\xd6\xc6\xdb (a INT)"
 
 
-class TestOptionalMetaData(base.PyMySQLReplicationTestCase):
+class TestOptionalMetaData(base.PyMySQLReplicationVersion8TestCase):
     def setUp(self):
         super(TestOptionalMetaData, self).setUp()
         self.stream.close()
@@ -1703,7 +1703,7 @@ def test_sync_drop_table_map_event_table_schema(self):
 
         event = self.stream.fetchone()
         self.assertIsInstance(event, TableMapEvent)
-        self.assertEqual(event.table_obj.data["columns"][0].name, "name")
+        self.assertEqual(event.table_obj.data["columns"][0].name, None)
         self.assertEqual(len(column_schemas), 0)
 
     def test_sync_column_drop_event_table_schema(self):
@@ -1734,9 +1734,9 @@ def test_sync_column_drop_event_table_schema(self):
         self.assertEqual(len(event.table_obj.data["columns"]), 3)
         self.assertEqual(column_schemas[0][0], "drop_column1")
         self.assertEqual(column_schemas[1][0], "drop_column3")
-        self.assertEqual(event.table_obj.data["columns"][0].name, "drop_column1")
-        self.assertEqual(event.table_obj.data["columns"][1].name, "drop_column2")
-        self.assertEqual(event.table_obj.data["columns"][2].name, "drop_column3")
+        self.assertEqual(event.table_obj.data["columns"][0].name, None)
+        self.assertEqual(event.table_obj.data["columns"][1].name, None)
+        self.assertEqual(event.table_obj.data["columns"][2].name, None)
 
     def tearDown(self):
         self.execute("SET GLOBAL binlog_row_metadata='MINIMAL';")

pymysqlreplication/tests/test_data_type.py

@@ -21,7 +21,7 @@
 from pymysqlreplication._compat import text_type
 
 
-__all__ = ["TestDataType"]
+__all__ = ["TestDataType", "TestDataTypeVersion8"]
 
 
 def to_binary_dict(d):
@@ -954,5 +954,49 @@ def test_varbinary(self):
             self.assertEqual(event.rows[0]["values"]["b"], b"\xff\x01\x00\x00")
 
 
+class TestDataTypeVersion8(base.PyMySQLReplicationVersion8TestCase):
+    def ignoredEvents(self):
+        return [GtidEvent, PreviousGtidsEvent]
+
+    def create_and_insert_value(self, create_query, insert_query):
+        self.execute(create_query)
+        self.execute(insert_query)
+        self.execute("COMMIT")
+
+        self.assertIsInstance(self.stream.fetchone(), RotateEvent)
+        self.assertIsInstance(self.stream.fetchone(), FormatDescriptionEvent)
+        # QueryEvent for the Create Table
+        self.assertIsInstance(self.stream.fetchone(), QueryEvent)
+
+        # QueryEvent for the BEGIN
+        self.assertIsInstance(self.stream.fetchone(), QueryEvent)
+
+        self.assertIsInstance(self.stream.fetchone(), TableMapEvent)
+
+        event = self.stream.fetchone()
+        if self.isMySQL56AndMore():
+            self.assertEqual(event.event_type, WRITE_ROWS_EVENT_V2)
+        else:
+            self.assertEqual(event.event_type, WRITE_ROWS_EVENT_V1)
+        self.assertIsInstance(event, WriteRowsEvent)
+        return event
+
+    def test_partition_id(self):
+        if not self.isMySQL80AndMore():
+            self.skipTest("Not supported in this version of MySQL")
+        create_query = "CREATE TABLE test (id INTEGER) \
+            PARTITION BY RANGE (id) ( \
+                PARTITION p0 VALUES LESS THAN (1),   \
+                PARTITION p1 VALUES LESS THAN (2),   \
+                PARTITION p2 VALUES LESS THAN (3),   \
+                PARTITION p3 VALUES LESS THAN (4),   \
+                PARTITION p4 VALUES LESS THAN (5)    \
+            )"
+        insert_query = "INSERT INTO test (id) VALUES(3)"
+        event = self.create_and_insert_value(create_query, insert_query)
+        self.assertEqual(event.extra_data_type, 1)
+        self.assertEqual(event.partition_id, 3)
+
+
 if __name__ == "__main__":
     unittest.main()

test.Dockerfile

@@ -19,6 +19,12 @@ ENV MYSQL_5_7_CTL ${MYSQL_5_7_CTL}
 ARG MYSQL_5_7_CTL_PORT
 ENV MYSQL_5_7_CTL_PORT ${MYSQL_5_7_CTL_PORT}
 
+ARG MYSQL_8_0
+ENV MYSQL_8_0 ${MYSQL_8_0}
+
+ARG MYSQL_8_0_PORT
+ENV MYSQL_8_0_PORT ${MYSQL_8_0_PORT}
+
 ARG MARIADB_10_6
 ENV MARIADB_10_6 ${MARIADB_10_6}