Revive automated testing

.github/ISSUE_TEMPLATE.md

@@ -3,8 +3,8 @@ Please specify the versions you are using. Exact version numbers are preferred.
 - Pymyrepl (e.g., 1.0.2):
 - OS (e.g., Ubuntu 18.04):
 - Database and version (Remove unnecessary options):
-  - MySQL: 
-  - MariaDB: 
+  - MySQL:
+  - MariaDB:
   - Percona:
 
 ### System Variables

.github/PULL_REQUEST_TEMPLATE.md

@@ -18,4 +18,3 @@
 
 ### Additional Information
 <!--If there's any additional information or context you'd like to provide for this PR, such as screenshots, reference documents, or release notes, please include them here.-->
-

.github/workflows/coverage.yml

@@ -1,13 +1,13 @@
 name: Python Coverage
-on: 
+on:
   push:
   pull_request:
   workflow_dispatch:
 jobs:
   build:
     runs-on: ubuntu-latest
     timeout-minutes: 3
-    
+
     steps:
       - name: Check out code
         uses: actions/checkout@v4

.github/workflows/pytest.yml

@@ -16,7 +16,8 @@ jobs:
           - {name: 'Pypy 3.7', python: 'pypy-3.7'}
           - {name: 'Pypy 3.10', python: 'pypy-3.10'}
     name: ${{ matrix.name }}
-    runs-on: ubuntu-latest
+    # ubuntu-latest does not have python 3.7
+    runs-on: ubuntu-22.04
     timeout-minutes: 3
 
     steps:
@@ -53,14 +54,10 @@ jobs:
         working-directory: pymysqlreplication/tests
         run: pytest -k "$PYTEST_SKIP_OPTION" --db=mysql-5
 
-      - name: Run tests for mysql-5-ctl
-        working-directory: pymysqlreplication/tests
-        run: pytest -k "$PYTEST_SKIP_OPTION" --db=mysql-5-ctl
-
       - name: Run tests for mysql-8
         working-directory: pymysqlreplication/tests
         run: pytest -k "$PYTEST_SKIP_OPTION" --db=mysql-8
 
       - name: Run tests for mariadb-10
         working-directory: pymysqlreplication/tests
-        run: pytest -k "$PYTEST_SKIP_OPTION" -m mariadb --db=mariadb-10
+        run: pytest -k "$PYTEST_SKIP_OPTION" --db=mariadb-10

.pre-commit-config.yaml

@@ -1,14 +1,14 @@
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v5.0.0
     hooks:
     -   id: trailing-whitespace
     -   id: end-of-file-fixer
     -   id: check-yaml
     -   id: check-added-large-files
 
 -   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.0.286
+    rev: v0.9.10
     hooks:
     -   id: ruff
         args: [ --fix, --exit-non-zero-on-fix ]

docker-compose-test.yml

@@ -1,5 +1,3 @@
-version: '3.4'
-
 x-mysql: &mysql
   environment:
     MYSQL_ALLOW_EMPTY_PASSWORD: true
@@ -21,36 +19,22 @@ x-mariadb: &mariadb
     --binlog-format=row
 
 services:
-  percona-5.7-ctl:
-    <<: *mysql
-    image: percona:5.7
-    ports:
-      - "3307:3306"
-    networks:
-      - default
-
   percona-5.7:
     <<: *mysql
     image: percona:5.7
-    ports:
-      - "3306:3306"
     networks:
       - default
 
   percona-8.0:
     <<: *mysql
     image: percona:8.0
     platform: linux/amd64
-    ports:
-      - "3309:3306"
     networks:
       - default
 
   mariadb-10.6:
     <<: *mariadb
     image: mariadb:10.6
-    ports:
-      - "3308:3306"
     volumes:
       - type: bind
         source: ./.mariadb
@@ -66,10 +50,7 @@ services:
       context: .
       dockerfile: test.Dockerfile
       args:
-        BASE_IMAGE: python:3.11-alpine
         MYSQL_5_7: percona-5.7
-        MYSQL_5_7_CTL: percona-5.7-ctl
-        MYSQL_5_7_CTL_PORT: 3306
         MYSQL_8_0: percona-8.0
         MYSQL_8_0_PORT: 3306
         MARIADB_10_6: mariadb-10.6
@@ -81,24 +62,34 @@ services:
       - |
         echo "wait mysql server"
 
-        while :
-        do
-          if mysql -h percona-5.7 --user=root --execute "SELECT version();" 2>&1 >/dev/null && mysql -h percona-5.7-ctl --user=root --execute "SELECT version();" 2>&1 >/dev/null && mysql -h percona-8.0 --user=root --execute "SELECT version();" 2>&1 >/dev/null; then
-            break
-          fi
+        check_servers_up() {
+          for s in percona-5.7 percona-8.0 mariadb-10.6; do
+            mariadb -h "$$s" --user=root --execute "SELECT version();" || {
+              echo "$$h not ready"
+              return 1
+            }
+          done
+          return 0
+        }
+
+        while ! check_servers_up; do
           sleep 1
         done
 
         echo "run pytest"
-        pytest -k "not test_no_trailing_rotate_event and not test_end_log_pos"
+        for db in mysql-5 mariadb-10 mysql-8; do
+          echo "testing with $$db"
+          pytest -vv -k "not test_no_trailing_rotate_event and not test_end_log_pos" --db "$$db"
+          echo "tested with $$db"
+        done
 
     working_dir: /pymysqlreplication
     networks:
       - default
     depends_on:
       - percona-5.7
-      - percona-5.7-ctl
       - percona-8.0
+      - mariadb-10.6
 
 networks:
   default:

examples/mysql_to_rabbitmq.py

@@ -16,13 +16,13 @@
     WriteRowsEvent,
 )
 
-MYSQL_SETTINGS = {"host": "127.0.0.1", "port": 3306, "user": "root", 
+MYSQL_SETTINGS = {"host": "127.0.0.1", "port": 3306, "user": "root",
                   "passwd": "password"}
 
 def main():
     stream = BinLogStreamReader(
         connection_settings=MYSQL_SETTINGS,
-        server_id=3,  
+        server_id=3,
         only_events=[DeleteRowsEvent, WriteRowsEvent, UpdateRowsEvent],
     )
 
@@ -31,14 +31,14 @@ def main():
         password='password'
     )
     params = pika.ConnectionParameters('rabbitmq_host', credentials=credentials)
-    
+
     # RabbitMQ Connection Settings
     conn = pika.BlockingConnection(params)
     channel = conn.channel()
     channel.queue_declare(queue='order')
     channel.exchange_declare(durable=True, exchange_type='direct', exchange='direct')
     channel.queue_bind(queue='order', exchange='direct', routing_key='order')
-    
+
     for binlogevent in stream:
         for row in binlogevent.rows:
             if isinstance(binlogevent, DeleteRowsEvent):
@@ -48,23 +48,23 @@ def main():
             elif isinstance(binlogevent, UpdateRowsEvent):
                 routing_key = "order"
                 message_body = row["after_values"].items()
-                
+
             elif isinstance(binlogevent, WriteRowsEvent):
                 routing_key = "order"
                 message_body = row["values"].items()
-           
-            properties = pika.BasicProperties(content_type='application/json', 
+
+            properties = pika.BasicProperties(content_type='application/json',
                                               delivery_mode=DeliveryMode.Transient)
             channel.basic_publish(
                 exchange='direct',
                 routing_key=routing_key,
                 body=json.dumps(message_body, default=lambda x: str(x)),
                 properties=properties
             )
-            
+
     stream.close()
     conn.close()
-    
+
 
 if __name__ == '__main__':
-    main()
+    main()

getting-started.md

@@ -1 +0,0 @@
-

pymysqlreplication/tests/base.py

@@ -45,8 +45,19 @@ def setUpDatabase(self, get_db):
     def setUp(self, charset="utf8"):
         # default
         self.conn_control = None
+
+        for i in ['host', 'port']:
+            env_override = self.database.pop(f'{i}_env_override', None)
+            if env_override:
+                override = os.environ.get(env_override)
+                if override:
+                    if i == 'port':
+                        override = int(override)
+                    self.database[i] = override
+
         db = copy.copy(self.database)
         db["db"] = None
+        db["charset"] = charset
         self.connect_conn_control(db)
         self.execute("DROP DATABASE IF EXISTS pymysqlreplication_test")
         self.execute("CREATE DATABASE pymysqlreplication_test")
@@ -55,8 +66,8 @@ def setUp(self, charset="utf8"):
         self.connect_conn_control(db)
         self.stream = None
         self.resetBinLog()
-        self.isMySQL56AndMore()
         self.__is_mariaDB = None
+        self.isMySQL56AndMore()
 
     def getMySQLVersion(self):
         """Return the MySQL version of the server
@@ -65,27 +76,37 @@ def getMySQLVersion(self):
         return self.execute("SELECT VERSION()").fetchone()[0].split("-")[0]
 
     def isMySQL56AndMore(self):
+        if self.isMariaDB():
+            return False
         version = float(self.getMySQLVersion().rsplit(".", 1)[0])
         if version >= 5.6:
             return True
         return False
 
     def isMySQL57(self):
+        if self.isMariaDB():
+            return False
         version = float(self.getMySQLVersion().rsplit(".", 1)[0])
         return version == 5.7
 
     def isMySQL80AndMore(self):
+        if self.isMariaDB():
+            return False
         version = float(self.getMySQLVersion().rsplit(".", 1)[0])
         return version >= 8.0
 
     def isMySQL8014AndMore(self):
+        if self.isMariaDB():
+            return False
         version = float(self.getMySQLVersion().rsplit(".", 1)[0])
         version_detail = int(self.getMySQLVersion().rsplit(".", 1)[1])
         if version > 8.0:
             return True
         return version == 8.0 and version_detail >= 14
 
     def isMySQL8016AndMore(self):
+        if self.isMariaDB():
+            return False
         version = float(self.getMySQLVersion().rsplit(".", 1)[0])
         version_detail = int(self.getMySQLVersion().rsplit(".", 1)[1])
         if version > 8.0:

pymysqlreplication/tests/config.json

@@ -5,34 +5,28 @@
     "passwd": "",
     "port": 3306,
     "use_unicode": true,
-    "charset": "utf8",
-    "db": "pymysqlreplication_test"
-  },
-  "mysql-5-ctl": {
-    "host": "localhost",
-    "user": "root",
-    "passwd": "",
-    "port": 3307,
-    "use_unicode": true,
-    "charset": "utf8",
-    "db": "pymysqlreplication_test"
+    "db": "pymysqlreplication_test",
+    "host_env_override": "MYSQL_5_7",
+    "port_env_override": "MYSQL_5_7_PORT"
   },
   "mariadb-10": {
     "host": "localhost",
     "user": "root",
     "passwd": "",
     "port": 3308,
     "use_unicode": true,
-    "charset": "utf8",
-    "db": "pymysqlreplication_test"
+    "db": "pymysqlreplication_test",
+    "host_env_override": "MARIADB_10_6",
+    "port_env_override": "MARIADB_10_6_PORT"
   },
   "mysql-8": {
     "host": "localhost",
     "user": "root",
     "passwd": "",
     "port": 3309,
     "use_unicode": true,
-    "charset": "utf8",
-    "db": "pymysqlreplication_test"
+    "db": "pymysqlreplication_test",
+    "host_env_override": "MYSQL_8_0",
+    "port_env_override": "MYSQL_8_0_PORT"
   }
 }