Merge branch 'julien-duponchelle:main' into main

Author: Samira-El

.github/workflows/release.yml

@@ -0,0 +1,31 @@
+name: Publish Python 🐍 distributions 📦 to PyPI and TestPyPI
+
+on: push
+
+jobs:
+  build-n-publish:
+    name: Build and publish Python 🐍 distributions 📦 to PyPI and TestPyPI
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.x"
+    - name: Install pypa/build
+      run: >-
+        python3 -m
+        pip install
+        build
+        --user
+    - name: Build a source tarball
+      run: >-
+        python3 -m
+        build
+        --sdist
+        --outdir dist/
+    - name: Publish distribution 📦 to PyPI
+      if: startsWith(github.ref, 'refs/tags')
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        password: ${{ secrets.PYPI_API_TOKEN }}

.travis.yml

@@ -3,10 +3,8 @@ language: python
 services:
   - docker
 python:
-  - "2.7"
   - "3.4"
-  - "3.5"
-  - "3.6"
+  - "3.11"
   - "pypy"
 env:
   - DB=mysql57

CHANGELOG

@@ -175,4 +175,15 @@
 * Handle null json
 
 0.31 12/03/2023
-* Fix parse error for query_events with MariaDB
+* Fix parse error for query_events with MariaDB
+
+0.40 07/05/2023
+* Drop support for Python 2.7
+* Gtid: remove __cmp__ due to python2 support dropped.
+* Mariadb 10.6.12: Mitigate corrupt binlog event bug
+
+0.41 03/06/2023
+* Zero-pad fixed-length binary fields
+
+0.42 25/06/2023
+* Add XAPrepareEvent, parse last_committed & sequence_number of GtidEvent

README.md

@@ -44,7 +44,6 @@ Project status
 
 The project is test with:
 * MySQL 5.5, 5.6 and 5.7
-* Python >= 2.7
 * Python 3.3, 3.4, 3.5 and 3.6 (3.2 is not supported)
 * PyPy (really faster than the standard Python interpreter)
 
@@ -332,14 +331,17 @@ Other contributors:
 * Dongwook Chan: Support for ZEROFILL, Correct timedelta value for negative MySQL TIME datatype, Fix parsing of row events for MySQL8 partitioned table, Parse status variables in query event, Parse status variables in query event , Fix parse errors with MariaDB (https://github.com/dongwook-chan)
 * Paul Vickers: Add support for specifying an end log_pos (https://github.com/paulvic)
 * Samira El Aabidi: Add support for MariaDB GTID (https://github.com/Samira-El)
-* Oliver Seemann: Handle large json, github actions (https://github.com/oseemann)
+* Oliver Seemann: Handle large json, github actions, 
+Zero-pad fixed-length binary fields (https://github.com/oseemann)
 * Mahadir Ahmad: Handle null json payload (https://github.com/mahadirz)
+* Axel Viala: Removal of Python 2.7 (https://github.com/darnuria)
+* Etern: Add XAPrepareEvent, parse last_committed & sequence_number of GtidEvent (https://github.com/etern)
 
 Thanks to GetResponse for their support
 
 Licence
 =======
-Copyright 2012-2022 Julien Duponchelle
+Copyright 2012-2023 Julien Duponchelle
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

docs/conf.py

@@ -41,16 +41,16 @@
 
 # General information about the project.
 project = u'Python MySQL Replication'
-copyright = u'2012-2021, Julien Duponchelle'
+copyright = u'2012-2023, Julien Duponchelle'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
-version = '0.31'
+version = '0.42'
 # The full version, including alpha/beta/rc tags.
-release = '0.31'
+release = '0.42'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

pymysqlreplication/binlogstream.py

@@ -319,6 +319,14 @@ def __connect_to_stream(self):
             cur.execute("set @master_heartbeat_period= %d" % heartbeat)
             cur.close()
 
+        # When replicating from Mariadb 10.6.12 using binlog coordinates, a slave capability < 4 triggers a bug in
+        # Mariadb, when it tries to replace GTID events with dummy ones. Given that this library understands GTID
+        # events, setting the capability to 4 circumvents this error.
+        # If the DB is mysql, this won't have any effect so no need to run this in a condition
+        cur = self._stream_connection.cursor()
+        cur.execute("SET @mariadb_slave_capability=4")
+        cur.close()
+
         self._register_slave()
 
         if not self.auto_position:
@@ -352,8 +360,6 @@ def __connect_to_stream(self):
             if self.is_mariadb:
                 # https://mariadb.com/kb/en/5-slave-registration/
                 cur = self._stream_connection.cursor()
-
-                cur.execute("SET @mariadb_slave_capability=4")
                 cur.execute("SET @slave_connect_state='%s'" % self.auto_position)
                 cur.execute("SET @slave_gtid_strict_mode=1")
                 cur.execute("SET @slave_gtid_ignore_duplicates=0")
@@ -610,7 +616,8 @@ def __get_table_information(self, schema, table):
                 cur.execute("""
                     SELECT
                         COLUMN_NAME, COLLATION_NAME, CHARACTER_SET_NAME,
-                        COLUMN_COMMENT, COLUMN_TYPE, COLUMN_KEY, ORDINAL_POSITION
+                        COLUMN_COMMENT, COLUMN_TYPE, COLUMN_KEY, ORDINAL_POSITION,
+                        DATA_TYPE, CHARACTER_OCTET_LENGTH
                     FROM
                         information_schema.columns
                     WHERE

pymysqlreplication/column.py

@@ -26,6 +26,12 @@ def __parse_column_definition(self, column_type, column_schema, packet):
         self.type_is_bool = False
         self.is_primary = column_schema["COLUMN_KEY"] == "PRI"
 
+        # Check for fixed-length binary type. When that's the case then we need
+        # to zero-pad the values to full length at read time.
+        self.fixed_binary_length = None
+        if column_schema["DATA_TYPE"] == "binary":
+            self.fixed_binary_length = column_schema["CHARACTER_OCTET_LENGTH"]
+
         if self.type == FIELD_TYPE.VARCHAR:
             self.max_length = struct.unpack('<H', packet.read(2))[0]
         elif self.type == FIELD_TYPE.DOUBLE:

pymysqlreplication/gtid.py

@@ -281,11 +281,6 @@ def decode(cls, payload):
             else '%d' % x
             for x in intervals])))
 
-    def __cmp__(self, other):
-        if other.sid != self.sid:
-            return cmp(self.sid, other.sid)
-        return cmp(self.intervals, other.intervals)
-
     def __eq__(self, other):
         if other.sid != self.sid:
             return False

pymysqlreplication/row_event.py

@@ -109,6 +109,7 @@ def _read_column_data(self,  cols_bitmap):
             name = self.table_map[self.table_id].columns[i].name
             unsigned = self.table_map[self.table_id].columns[i].unsigned
             zerofill = self.table_map[self.table_id].columns[i].zerofill
+            fixed_binary_length = self.table_map[self.table_id].columns[i].fixed_binary_length
 
             if BitGet(cols_bitmap, i) == 0:
                 values[name] = None
@@ -154,6 +155,14 @@ def _read_column_data(self,  cols_bitmap):
                     values[name] = self.__read_string(2, column)
                 else:
                     values[name] = self.__read_string(1, column)
+
+                if fixed_binary_length and len(values[name]) < fixed_binary_length:
+                    # Fixed-length binary fields are stored in the binlog
+                    # without trailing zeros and must be padded with zeros up
+                    # to the specified length at read time.
+                    nr_pad = fixed_binary_length - len(values[name])
+                    values[name] += b'\x00' * nr_pad
+
             elif column.type == FIELD_TYPE.NEWDECIMAL:
                 values[name] = self.__read_new_decimal(column)
             elif column.type == FIELD_TYPE.BLOB:
@@ -640,6 +649,8 @@ def __init__(self, from_packet, event_size, table_map, ctl_connection, **kwargs)
                         'COLUMN_NAME': '__dropped_col_{i}__'.format(i=i),
                         'COLLATION_NAME': None,
                         'CHARACTER_SET_NAME': None,
+                        'CHARACTER_OCTET_LENGTH': None,
+                        'DATA_TYPE': 'BLOB',
                         'COLUMN_COMMENT': None,
                         'COLUMN_TYPE': 'BLOB',  # we don't know what it is, so let's not do anything with it.
                         'COLUMN_KEY': '',

pymysqlreplication/tests/test_data_objects.py

@@ -22,6 +22,8 @@ def test_column_is_primary(self):
                      {"COLUMN_NAME": "test",
                       "COLLATION_NAME": "utf8_general_ci",
                       "CHARACTER_SET_NAME": "UTF8",
+                      "CHARACTER_OCTET_LENGTH": None,
+                      "DATA_TYPE": "tinyint",
                       "COLUMN_COMMENT": "",
                       "COLUMN_TYPE": "tinyint(2)",
                       "COLUMN_KEY": "PRI"},
@@ -33,6 +35,8 @@ def test_column_not_primary(self):
                      {"COLUMN_NAME": "test",
                       "COLLATION_NAME": "utf8_general_ci",
                       "CHARACTER_SET_NAME": "UTF8",
+                      "CHARACTER_OCTET_LENGTH": None,
+                      "DATA_TYPE": "tinyint",
                       "COLUMN_COMMENT": "",
                       "COLUMN_TYPE": "tinyint(2)",
                       "COLUMN_KEY": ""},
@@ -44,6 +48,8 @@ def test_column_serializable(self):
                      {"COLUMN_NAME": "test",
                       "COLLATION_NAME": "utf8_general_ci",
                       "CHARACTER_SET_NAME": "UTF8",
+                      "CHARACTER_OCTET_LENGTH": None,
+                      "DATA_TYPE": "tinyint",
                       "COLUMN_COMMENT": "",
                       "COLUMN_TYPE": "tinyint(2)",
                       "COLUMN_KEY": "PRI"},

pymysqlreplication/tests/test_data_type.py

@@ -101,6 +101,18 @@ def create_and_get_tablemap_event(self, bit):
 
         return event
 
+    def test_varbinary(self):
+        create_query = "CREATE TABLE test(b VARBINARY(4))"
+        insert_query = "INSERT INTO test VALUES(UNHEX('ff010000'))"
+        event = self.create_and_insert_value(create_query, insert_query)
+        self.assertEqual(event.rows[0]["values"]["b"], b'\xff\x01\x00\x00')
+
+    def test_fixed_length_binary(self):
+        create_query = "CREATE TABLE test(b BINARY(4))"
+        insert_query = "INSERT INTO test VALUES(UNHEX('ff010000'))"
+        event = self.create_and_insert_value(create_query, insert_query)
+        self.assertEqual(event.rows[0]["values"]["b"], b'\xff\x01\x00\x00')
+
     def test_decimal(self):
         create_query = "CREATE TABLE test (test DECIMAL(2,1))"
         insert_query = "INSERT INTO test VALUES(4.2)"

setup.py

@@ -6,14 +6,9 @@
 except ImportError:
     from distutils.core import setup, Command
 
+from pathlib import Path
 import sys
 
-tests_require = []
-
-# add unittest2 to tests_require for python < 2.7
-if sys.version_info < (2, 7):
-    tests_require.append("unittest2")
-
 
 class TestCommand(Command):
     user_options = []
@@ -34,21 +29,25 @@ def run(self):
         unittest.main(tests, argv=sys.argv[:1])
 
 
-version = "0.31"
+version = "0.42"
+
+this_directory = Path(__file__).parent
+long_description = (this_directory / "README.md").read_text()
 
 setup(
     name="mysql-replication",
     version=version,
-    url="https://github.com/noplay/python-mysql-replication",
+    url="https://github.com/julien-duponchelle/python-mysql-replication",
     author="Julien Duponchelle",
     author_email="[email protected]",
     description=("Pure Python Implementation of MySQL replication protocol "
                  "build on top of PyMYSQL."),
+    long_description=long_description,
+    long_description_content_type='text/markdown',
     license="Apache 2",
     packages=["pymysqlreplication",
               "pymysqlreplication.constants",
               "pymysqlreplication.tests"],
     cmdclass={"test": TestCommand},
-    extras_require={'test': tests_require},
     install_requires=['pymysql>=0.10'],
 )