From 8baa8ab5ed0f3f3ec45e1a3261165f7e561721d8 Mon Sep 17 00:00:00 2001 From: will-k Date: Thu, 24 Aug 2023 07:20:10 +0900 Subject: [PATCH 1/9] Eliminate `log_slave_updates` system variable, closes #410 - `log_slave_updates` modifies to which file binlog events is written. - The variable is not applicable to python-mysql-replication, because no file is written --- docker-compose-test.yml | 4 ++-- docker-compose.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docker-compose-test.yml b/docker-compose-test.yml index 38d95827..45b0de43 100644 --- a/docker-compose-test.yml +++ b/docker-compose-test.yml @@ -8,7 +8,7 @@ services: MYSQL_DATABASE: pymysqlreplication_test ports: - 3306:3306 - command: mysqld --log-bin=mysql-bin.log --server-id 1 --binlog-format=row --gtid_mode=on --enforce-gtid-consistency=on --log_slave_updates + command: mysqld --log-bin=mysql-bin.log --server-id 1 --binlog-format=row --gtid_mode=on --enforce-gtid-consistency=on restart: always networks: - default @@ -20,7 +20,7 @@ services: MYSQL_DATABASE: pymysqlreplication_test ports: - 3307:3307 - command: mysqld --log-bin=mysql-bin.log --server-id 1 --binlog-format=row --gtid_mode=on --enforce-gtid-consistency=on --log_slave_updates -P 3307 + command: mysqld --log-bin=mysql-bin.log --server-id 1 --binlog-format=row --gtid_mode=on --enforce-gtid-consistency=on -P 3307 pymysqlreplication: build: diff --git a/docker-compose.yml b/docker-compose.yml index 45b53c3d..5c74f7bc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,7 @@ services: MYSQL_ALLOW_EMPTY_PASSWORD: true ports: - 3306:3306 - command: mysqld --log-bin=mysql-bin.log --server-id 1 --binlog-format=row --gtid_mode=on --enforce-gtid-consistency=on --log_slave_updates + command: mysqld --log-bin=mysql-bin.log --server-id 1 --binlog-format=row --gtid_mode=on --enforce-gtid-consistency=on percona-5.7-ctl: image: percona:5.7 @@ -14,7 +14,7 @@ services: MYSQL_ALLOW_EMPTY_PASSWORD: true ports: - 3307:3307 - command: mysqld --log-bin=mysql-bin.log --server-id 1 --binlog-format=row --gtid_mode=on --enforce-gtid-consistency=on --log_slave_updates -P 3307 + command: mysqld --log-bin=mysql-bin.log --server-id 1 --binlog-format=row --gtid_mode=on --enforce-gtid-consistency=on -P 3307 mariadb-10.6: image: mariadb:10.6 From 53e7c2de5094566ebfc0146bcd4f4c6bddf746fc Mon Sep 17 00:00:00 2001 From: will-k Date: Thu, 24 Aug 2023 08:04:14 +0900 Subject: [PATCH 2/9] Refactor docker-compose.yml to use extension and YAML anchors for improved readability and expandability Changes: - Updated version to 3.4 at which extension for docker compose was first introduced - Use extension to hold common fields and anchors to reference the extensions --- docker-compose-test.yml | 49 +++++++++++++++++++++++++++------------ docker-compose.yml | 51 ++++++++++++++++++++++------------------- 2 files changed, 61 insertions(+), 39 deletions(-) diff --git a/docker-compose-test.yml b/docker-compose-test.yml index 45b0de43..20d4efbf 100644 --- a/docker-compose-test.yml +++ b/docker-compose-test.yml @@ -1,26 +1,45 @@ -version: '3.2' +version: '3.4' + +x-mysql: &mysql + environment: + MYSQL_ALLOW_EMPTY_PASSWORD: true + command: > + mysqld + --log-bin=mysql-bin.log + --server-id 1 + --binlog-format=row + --gtid_mode=on + --enforce-gtid-consistency=on + --log_slave_updates + +x-mariadb: &mariadb + environment: + MARIADB_ALLOW_EMPTY_ROOT_PASSWORD: 1 + command: | + --server-id=1 + --default-authentication-plugin=mysql_native_password + --log-bin=master-bin + --binlog-format=row + --log-slave-updates=on + services: percona-5.7: - platform: linux/amd64 + <<: *mysql image: percona:5.7 - environment: - MYSQL_ALLOW_EMPTY_PASSWORD: true - MYSQL_DATABASE: pymysqlreplication_test ports: - - 3306:3306 - command: mysqld --log-bin=mysql-bin.log --server-id 1 --binlog-format=row --gtid_mode=on --enforce-gtid-consistency=on - restart: always - networks: - - default + - "3306:3306" percona-5.7-ctl: + <<: *mysql image: percona:5.7 - environment: - MYSQL_ALLOW_EMPTY_PASSWORD: true - MYSQL_DATABASE: pymysqlreplication_test ports: - - 3307:3307 - command: mysqld --log-bin=mysql-bin.log --server-id 1 --binlog-format=row --gtid_mode=on --enforce-gtid-consistency=on -P 3307 + - "3307:3306" + + mariadb-10.6: + <<: *mariadb + image: mariadb:10.6 + ports: + - "3308:3306" pymysqlreplication: build: diff --git a/docker-compose.yml b/docker-compose.yml index 5c74f7bc..a7c9c181 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,37 +1,40 @@ -version: '3.2' +version: '3.4' + +x-mysql: &mysql + environment: + MYSQL_ALLOW_EMPTY_PASSWORD: true + command: > + mysqld + --log-bin=mysql-bin.log + --server-id 1 + --binlog-format=row + --gtid_mode=on + --enforce-gtid-consistency=on + +x-mariadb: &mariadb + environment: + MARIADB_ALLOW_EMPTY_ROOT_PASSWORD: 1 + command: | + --log-bin=master-bin + --server-id=1 + --default-authentication-plugin=mysql_native_password + --binlog-format=row + services: percona-5.7: + <<: *mysql image: percona:5.7 - environment: - MYSQL_ALLOW_EMPTY_PASSWORD: true ports: - - 3306:3306 - command: mysqld --log-bin=mysql-bin.log --server-id 1 --binlog-format=row --gtid_mode=on --enforce-gtid-consistency=on + - "3306:3306" percona-5.7-ctl: + <<: *mysql image: percona:5.7 - environment: - MYSQL_ALLOW_EMPTY_PASSWORD: true ports: - - 3307:3307 - command: mysqld --log-bin=mysql-bin.log --server-id 1 --binlog-format=row --gtid_mode=on --enforce-gtid-consistency=on -P 3307 + - "3307:3306" mariadb-10.6: + <<: *mariadb image: mariadb:10.6 - environment: - MARIADB_ALLOW_EMPTY_ROOT_PASSWORD: 1 ports: - "3308:3306" - command: | - --server-id=1 - --default-authentication-plugin=mysql_native_password - --log-bin=master-bin - --binlog-format=row - --log-slave-updates=on - volumes: - - type: bind - source: ./.mariadb - target: /opt/key_file - - type: bind - source: ./.mariadb/my.cnf - target: /etc/mysql/my.cnf From 7e8a62bd84540d0e91ab03d2c214336ad0af1010 Mon Sep 17 00:00:00 2001 From: Soul Lee Date: Fri, 25 Aug 2023 01:26:51 +0900 Subject: [PATCH 3/9] fix: use folded block scalar --- docker-compose-test.yml | 2 +- docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose-test.yml b/docker-compose-test.yml index 20d4efbf..06af6b21 100644 --- a/docker-compose-test.yml +++ b/docker-compose-test.yml @@ -15,7 +15,7 @@ x-mysql: &mysql x-mariadb: &mariadb environment: MARIADB_ALLOW_EMPTY_ROOT_PASSWORD: 1 - command: | + command: > --server-id=1 --default-authentication-plugin=mysql_native_password --log-bin=master-bin diff --git a/docker-compose.yml b/docker-compose.yml index a7c9c181..51de152b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,7 +14,7 @@ x-mysql: &mysql x-mariadb: &mariadb environment: MARIADB_ALLOW_EMPTY_ROOT_PASSWORD: 1 - command: | + command: > --log-bin=master-bin --server-id=1 --default-authentication-plugin=mysql_native_password From 83b81fa2f17622db5ddca52097b42d904cc1275b Mon Sep 17 00:00:00 2001 From: Soul Lee Date: Fri, 25 Aug 2023 01:28:36 +0900 Subject: [PATCH 4/9] fix: add volumes to test MariadbStartEncryptionEvent --- docker-compose-test.yml | 7 +++++++ docker-compose.yml | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/docker-compose-test.yml b/docker-compose-test.yml index 06af6b21..c37c1f1f 100644 --- a/docker-compose-test.yml +++ b/docker-compose-test.yml @@ -40,6 +40,13 @@ services: image: mariadb:10.6 ports: - "3308:3306" + volumes: + - type: bind + source: ./.mariadb + target: /opt/key_file + - type: bind + source: ./.mariadb/my.cnf + target: /etc/mysql/my.cnf pymysqlreplication: build: diff --git a/docker-compose.yml b/docker-compose.yml index 51de152b..9e68758c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -38,3 +38,10 @@ services: image: mariadb:10.6 ports: - "3308:3306" + volumes: + - type: bind + source: ./.mariadb + target: /opt/key_file + - type: bind + source: ./.mariadb/my.cnf + target: /etc/mysql/my.cnf From 261fb67236188a4a794f58b13ffe5a909c5c961b Mon Sep 17 00:00:00 2001 From: lre12 Date: Fri, 25 Aug 2023 10:52:47 +0900 Subject: [PATCH 5/9] fix: docker-compose-test mysql server check query --- docker-compose-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose-test.yml b/docker-compose-test.yml index c37c1f1f..4d8c49a7 100644 --- a/docker-compose-test.yml +++ b/docker-compose-test.yml @@ -65,7 +65,7 @@ services: while : do - if mysql -h percona-5.7 --user=root --execute "USE pymysqlreplication_test;" 2>&1 >/dev/null && mysql -h percona-5.7-ctl --port=3307 --user=root --execute "USE pymysqlreplication_test;" 2>&1 >/dev/null; then + if mysql -h percona-5.7 --user=root --execute "SELECT version();" 2>&1 >/dev/null && mysql -h percona-5.7-ctl --port=3307 --user=root --execute "USE pymysqlreplication_test;" 2>&1 >/dev/null; then break fi sleep 1 From 994d1cda8854af384a556e9f1b9bc8125e4f5009 Mon Sep 17 00:00:00 2001 From: will-k Date: Fri, 25 Aug 2023 11:34:44 +0900 Subject: [PATCH 6/9] Eliminate log_slave_updates system variable in docker-compose-test.yml --- docker-compose-test.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker-compose-test.yml b/docker-compose-test.yml index c37c1f1f..81950432 100644 --- a/docker-compose-test.yml +++ b/docker-compose-test.yml @@ -10,7 +10,6 @@ x-mysql: &mysql --binlog-format=row --gtid_mode=on --enforce-gtid-consistency=on - --log_slave_updates x-mariadb: &mariadb environment: @@ -20,7 +19,6 @@ x-mariadb: &mariadb --default-authentication-plugin=mysql_native_password --log-bin=master-bin --binlog-format=row - --log-slave-updates=on services: percona-5.7: From 703349509e9f643c0dab8413d186ae04b1918067 Mon Sep 17 00:00:00 2001 From: lre12 Date: Sat, 26 Aug 2023 10:37:23 +0900 Subject: [PATCH 7/9] fix: docker-compose settings for mariadb --- docker-compose-test.yml | 22 ++++++++++++++++------ pymysqlreplication/tests/base.py | 4 ++-- pymysqlreplication/tests/test_basic.py | 5 ++--- test.Dockerfile | 11 ++++++++++- 4 files changed, 30 insertions(+), 12 deletions(-) diff --git a/docker-compose-test.yml b/docker-compose-test.yml index b70e6b10..3634e304 100644 --- a/docker-compose-test.yml +++ b/docker-compose-test.yml @@ -21,17 +21,21 @@ x-mariadb: &mariadb --binlog-format=row services: - percona-5.7: + percona-5.7-ctl: <<: *mysql image: percona:5.7 ports: - - "3306:3306" + - "3307:3306" + networks: + - default - percona-5.7-ctl: + percona-5.7: <<: *mysql image: percona:5.7 ports: - - "3307:3306" + - "3306:3306" + networks: + - default mariadb-10.6: <<: *mariadb @@ -45,6 +49,8 @@ services: - type: bind source: ./.mariadb/my.cnf target: /etc/mysql/my.cnf + networks: + - default pymysqlreplication: build: @@ -54,6 +60,9 @@ services: BASE_IMAGE: python:3.11-alpine MYSQL_5_7: percona-5.7 MYSQL_5_7_CTL: percona-5.7-ctl + MYSQL_5_7_CTL_PORT: 3306 + MARIADB_10_6: mariadb-10.6 + MARIADB_10_6_PORT: 3306 command: - /bin/sh @@ -63,7 +72,7 @@ services: while : do - if mysql -h percona-5.7 --user=root --execute "SELECT version();" 2>&1 >/dev/null && mysql -h percona-5.7-ctl --port=3307 --user=root --execute "USE pymysqlreplication_test;" 2>&1 >/dev/null; then + if mysql -h percona-5.7 --user=root --execute "SELECT version();" 2>&1 >/dev/null && mysql -h percona-5.7-ctl --user=root --execute "SELECT version();" 2>&1 >/dev/null; then break fi sleep 1 @@ -80,4 +89,5 @@ services: - percona-5.7-ctl networks: - default: {} + default: + driver: bridge diff --git a/pymysqlreplication/tests/base.py b/pymysqlreplication/tests/base.py index 009c3e29..6cb23094 100644 --- a/pymysqlreplication/tests/base.py +++ b/pymysqlreplication/tests/base.py @@ -125,10 +125,10 @@ class PyMySQLReplicationMariaDbTestCase(PyMySQLReplicationTestCase): def setUp(self): # default self.database = { - "host": "localhost", + "host": os.environ.get("MARIADB_10_6") or "localhost", "user": "root", "passwd": "", - "port": 3308, + "port": int(os.environ.get("MARIADB_10_6_PORT") or 3308), "use_unicode": True, "charset": "utf8", "db": "pymysqlreplication_test" diff --git a/pymysqlreplication/tests/test_basic.py b/pymysqlreplication/tests/test_basic.py index 32b237e0..d6ab5bee 100644 --- a/pymysqlreplication/tests/test_basic.py +++ b/pymysqlreplication/tests/test_basic.py @@ -771,9 +771,8 @@ def setUp(self): self.stream.close() ctl_db = copy.copy(self.database) ctl_db["db"] = None - ctl_db["port"] = 3307 - if os.environ.get("MYSQL_5_7_CTL") is not None: - ctl_db["host"] = os.environ.get("MYSQL_5_7_CTL") + ctl_db["port"] = int(os.environ.get("MYSQL_5_7_CTL_PORT")) or 3307 + ctl_db["host"] = os.environ.get("MYSQL_5_7_CTL") or "localhost" self.ctl_conn_control = pymysql.connect(**ctl_db) self.ctl_conn_control.cursor().execute("DROP DATABASE IF EXISTS pymysqlreplication_test") self.ctl_conn_control.cursor().execute("CREATE DATABASE pymysqlreplication_test") diff --git a/test.Dockerfile b/test.Dockerfile index 6aa2e34a..efb992db 100644 --- a/test.Dockerfile +++ b/test.Dockerfile @@ -13,4 +13,13 @@ ARG MYSQL_5_7 ENV MYSQL_5_7 ${MYSQL_5_7} ARG MYSQL_5_7_CTL -ENV MYSQL_5_7_CTL ${MYSQL_5_7_CTL} \ No newline at end of file +ENV MYSQL_5_7_CTL ${MYSQL_5_7_CTL} + +ARG MYSQL_5_7_CTL_PORT +ENV MYSQL_5_7_CTL_PORT ${MYSQL_5_7_CTL_PORT} + +ARG MARIADB_10_6 +ENV MARIADB_10_6 ${MARIADB_10_6} + +ARG MARIADB_10_6_PORT +ENV MARIADB_10_6_PORT ${MARIADB_10_6_PORT} \ No newline at end of file From ffbe2a438db45a1059442b84ef65ec791c59106b Mon Sep 17 00:00:00 2001 From: lre12 Date: Sat, 26 Aug 2023 13:39:57 +0900 Subject: [PATCH 8/9] fix: MYSQL_5_7_CTL_PORT casting --- pymysqlreplication/tests/test_basic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymysqlreplication/tests/test_basic.py b/pymysqlreplication/tests/test_basic.py index d6ab5bee..ed6c62bd 100644 --- a/pymysqlreplication/tests/test_basic.py +++ b/pymysqlreplication/tests/test_basic.py @@ -771,7 +771,7 @@ def setUp(self): self.stream.close() ctl_db = copy.copy(self.database) ctl_db["db"] = None - ctl_db["port"] = int(os.environ.get("MYSQL_5_7_CTL_PORT")) or 3307 + ctl_db["port"] = int(os.environ.get("MYSQL_5_7_CTL_PORT") or 3307) ctl_db["host"] = os.environ.get("MYSQL_5_7_CTL") or "localhost" self.ctl_conn_control = pymysql.connect(**ctl_db) self.ctl_conn_control.cursor().execute("DROP DATABASE IF EXISTS pymysqlreplication_test") From f0c43c627d28184bad2bcdf22259cde0b3462dc4 Mon Sep 17 00:00:00 2001 From: lre12 Date: Sat, 26 Aug 2023 13:50:24 +0900 Subject: [PATCH 9/9] add copy .mariadb --- test.Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/test.Dockerfile b/test.Dockerfile index efb992db..0272e00f 100644 --- a/test.Dockerfile +++ b/test.Dockerfile @@ -1,6 +1,7 @@ ARG BASE_IMAGE=${BASE_IMAGE:-python:3.11-alpine} FROM ${BASE_IMAGE} +COPY .mariadb .mariadb COPY pymysqlreplication pymysqlreplication COPY README.md README.md COPY setup.py setup.py