Developed the 'MariadbStartEncryptionEvent' (julien-duponchelle#415)

* implement StartEncryptionEvent

* Completed setup of MariaDB test environment.

* Fix: Correct assertion in test_basic:_allowed_event_list

* Fixed docker-compose.yaml

* Fix: Correct assertion in test_basic:_allowed_event_list

* Add unit tests for MariadbStartEncryptionEvent class

* Fix:find file from current directroy

* Fix: find the file path

* Rename StartEncryptionEvent to MariadbStartEncryptionEvent

* add description for MariadbStartEncruptionEvent

* fix description for MariadbStartEncryptionEvent

---------

Co-authored-by: Pilmo <[email protected]>
Co-authored-by: BeautterLife <[email protected]>

Author: 3 people

.mariadb/my.cnf

@@ -0,0 +1,23 @@
+[client-server]
+# Port or socket location where to connect
+# port = 3306
+socket = /run/mysqld/mysqld.sock
+
+# Import all .cnf files from configuration directory
+
+!includedir /etc/mysql/mariadb.conf.d/
+!includedir /etc/mysql/conf.d/
+
+
+[mariadb]
+plugin_load_add = file_key_management
+# Key files that are not encrypted
+loose_file_key_management_filename = /opt/key_file/no_encryption_key.key
+
+# Encrypted key file
+# loose_file_key_management_filename=/opt/key_file/keyfile.enc
+# loose_file_key_management_filekey=FILE:/opt/key_file/no_encryption_key.key
+# file_key_management_encryption_algorithm=aes_ctr
+
+# Set encrypt_binlog
+encrypt_binlog=ON

.mariadb/no_encryption_key.key

@@ -0,0 +1 @@
+1;dda0ccb18a28b0b4c2448b5f0217a134

docker-compose.yml

@@ -28,4 +28,10 @@ services:
       --log-bin=master-bin
       --binlog-format=row
       --log-slave-updates=on
-      
+    volumes:
+      - type: bind
+        source: ./.mariadb
+        target: /opt/key_file
+      - type: bind
+        source: ./.mariadb/my.cnf
+        target: /etc/mysql/my.cnf  

pymysqlreplication/binlogstream.py

@@ -15,7 +15,7 @@
     XidEvent, GtidEvent, StopEvent, XAPrepareEvent,
     BeginLoadQueryEvent, ExecuteLoadQueryEvent,
     HeartbeatLogEvent, NotImplementedEvent, MariadbGtidEvent,
-    MariadbAnnotateRowsEvent, RandEvent)
+    MariadbAnnotateRowsEvent, RandEvent, MariadbStartEncryptionEvent)
 from .exceptions import BinLogNotEnabled
 from .row_event import (
     UpdateRowsEvent, WriteRowsEvent, DeleteRowsEvent, TableMapEvent)
@@ -622,7 +622,8 @@ def _allowed_event_list(self, only_events, ignored_events,
                 NotImplementedEvent,
                 MariadbGtidEvent,
                 MariadbAnnotateRowsEvent,
-                RandEvent
+                RandEvent,
+                MariadbStartEncryptionEvent
                 ))
         if ignored_events is not None:
             for e in ignored_events:

pymysqlreplication/event.py

@@ -488,6 +488,35 @@ def _dump(self):
         print("seed1: %d" % (self.seed1))
         print("seed2: %d" % (self.seed2))
 
+class MariadbStartEncryptionEvent(BinLogEvent):
+    """
+    Since MariaDB 10.1.7, 
+    the START_ENCRYPTION event is written to every binary log file 
+    if encrypt_binlog is set to ON. Prior to enabling this setting, 
+    additional configuration steps are required in MariaDB. 
+    (Link: https://mariadb.com/kb/en/encrypting-binary-logs/)
+
+    This event is written just once, after the Format Description event
+
+    Attributes:
+        schema: The Encryption scheme, always set to 1 for system files.
+        key_version: The Encryption key version.
+        nonce: Nonce (12 random bytes) of current binlog file.
+    """
+
+    def __init__(self, from_packet, event_size, table_map, ctl_connection, **kwargs):
+        super(MariadbStartEncryptionEvent, self).__init__(from_packet, event_size, table_map, ctl_connection, **kwargs)
+
+        self.schema = self.packet.read_uint8()
+        self.key_version = self.packet.read_uint32()
+        self.nonce = self.packet.read(12)
+
+    def _dump(self):
+        print("Schema: %d" % self.schema)
+        print("Key version: %d" % self.key_version)
+        print(f"Nonce: {self.nonce}")
+
+
 class NotImplementedEvent(BinLogEvent):
     def __init__(self, from_packet, event_size, table_map, ctl_connection, **kwargs):
         super(NotImplementedEvent, self).__init__(

pymysqlreplication/packet.py

@@ -88,7 +88,7 @@ class BinLogPacketWrapper(object):
         constants.MARIADB_BINLOG_CHECKPOINT_EVENT: event.NotImplementedEvent,
         constants.MARIADB_GTID_EVENT: event.MariadbGtidEvent,
         constants.MARIADB_GTID_GTID_LIST_EVENT: event.NotImplementedEvent,
-        constants.MARIADB_START_ENCRYPTION_EVENT: event.NotImplementedEvent
+        constants.MARIADB_START_ENCRYPTION_EVENT: event.MariadbStartEncryptionEvent
     }
 
     def __init__(self, from_packet, table_map,

pymysqlreplication/tests/test_basic.py

@@ -18,6 +18,7 @@
 from pymysqlreplication.exceptions import TableMetadataUnavailableError
 from pymysqlreplication.constants.BINLOG import *
 from pymysqlreplication.row_event import *
+from pathlib import Path
 
 __all__ = ["TestBasicBinLogStreamReader", "TestMultipleRowBinLogStreamReader", "TestCTLConnectionSettings", "TestGtidBinLogStreamReader", "TestMariadbBinlogStreamReader", "TestStatementConnectionSetting"]
 
@@ -27,9 +28,9 @@ def ignoredEvents(self):
         return [GtidEvent]
 
     def test_allowed_event_list(self):
-        self.assertEqual(len(self.stream._allowed_event_list(None, None, False)), 18)
-        self.assertEqual(len(self.stream._allowed_event_list(None, None, True)), 17)
-        self.assertEqual(len(self.stream._allowed_event_list(None, [RotateEvent], False)), 17)
+        self.assertEqual(len(self.stream._allowed_event_list(None, None, False)), 19)
+        self.assertEqual(len(self.stream._allowed_event_list(None, None, True)), 18)
+        self.assertEqual(len(self.stream._allowed_event_list(None, [RotateEvent], False)), 18)
         self.assertEqual(len(self.stream._allowed_event_list([RotateEvent], None, False)), 1)
 
     def test_read_query_event(self):
@@ -1036,6 +1037,42 @@ def test_annotate_rows_event(self):
         self.assertEqual(event.sql_statement,insert_query)
         self.assertIsInstance(event,MariadbAnnotateRowsEvent)
 
+    def test_start_encryption_event(self):
+        query = "CREATE TABLE test (id INT NOT NULL AUTO_INCREMENT, data VARCHAR (50) NOT NULL, PRIMARY KEY (id))"
+        self.execute(query)
+        query = "INSERT INTO test (data) VALUES('Hello World')"
+        self.execute(query)
+        self.execute("COMMIT")
+
+        self.assertIsInstance(self.stream.fetchone(), RotateEvent)
+        self.assertIsInstance(self.stream.fetchone(), FormatDescriptionEvent)
+
+        start_encryption_event = self.stream.fetchone()
+        self.assertIsInstance(start_encryption_event, MariadbStartEncryptionEvent)
+
+        schema = start_encryption_event.schema
+        key_version = start_encryption_event.key_version
+        nonce = start_encryption_event.nonce
+
+        from pathlib import Path
+
+        encryption_key_file_path = Path(__file__).parent.parent.parent
+
+        try:
+            with open(f"{encryption_key_file_path}/.mariadb/no_encryption_key.key", "r") as key_file:
+                first_line = key_file.readline()
+                key_version_from_key_file = int(first_line.split(";")[0])
+        except Exception as e:
+            self.fail("raised unexpected exception: {exception}".format(exception=e))
+        finally:
+            self.resetBinLog()
+
+        # schema is always 1
+        self.assertEqual(schema, 1)
+        self.assertEqual(key_version, key_version_from_key_file)
+        self.assertEqual(type(nonce), bytes)
+        self.assertEqual(len(nonce), 12)        
+        
 class TestStatementConnectionSetting(base.PyMySQLReplicationTestCase):
     def setUp(self):
         super(TestStatementConnectionSetting, self).setUp()
@@ -1065,8 +1102,8 @@ def test_rand_event(self):
     def tearDown(self):
         self.execute("SET @@binlog_format='ROW'")
         self.assertEqual(self.bin_log_format(), "ROW")
-        super(TestStatementConnectionSetting, self).tearDown()
-        
+        super(TestStatementConnectionSetting, self).tearDown()        
+
 
 if __name__ == "__main__":
     import unittest