Skip to content

Commit 4b19633

Browse files
suhwan-cheonsuhwan-cheon
authored
Merge branch 'main' into feature/previous-gtid-event
2 parents 77768ac + a19a5a5 commit 4b19633

16 files changed

+858
-252
lines changed

‎.github/workflows/pytest.yml

+11-3
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,17 @@ jobs:
2828
run: |
2929
docker compose create
3030
docker compose start
31-
# Wait for the services to accept connections,
32-
# TODO: do that smarter, poll connection attempt until it succeeds
33-
sleep 30
31+
echo "wait mysql server"
32+
33+
while :
34+
do
35+
if mysql -h 127.0.0.1 --user=root --execute "SELECT version();" 2>&1 >/dev/null && mysql -h 127.0.0.1 --port=3307 --user=root --execute "SELECT version();" 2>&1 >/dev/null; then
36+
break
37+
fi
38+
sleep 1
39+
done
40+
41+
echo "run pytest"
3442
3543
- name: Install dependencies
3644
run: |

‎.mariadb/my.cnf

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
[client-server]
2+
# Port or socket location where to connect
3+
# port = 3306
4+
socket = /run/mysqld/mysqld.sock
5+
6+
# Import all .cnf files from configuration directory
7+
8+
!includedir /etc/mysql/mariadb.conf.d/
9+
!includedir /etc/mysql/conf.d/
10+
11+
12+
[mariadb]
13+
plugin_load_add = file_key_management
14+
# Key files that are not encrypted
15+
loose_file_key_management_filename = /opt/key_file/no_encryption_key.key
16+
17+
# Encrypted key file
18+
# loose_file_key_management_filename=/opt/key_file/keyfile.enc
19+
# loose_file_key_management_filekey=FILE:/opt/key_file/no_encryption_key.key
20+
# file_key_management_encryption_algorithm=aes_ctr
21+
22+
# Set encrypt_binlog
23+
encrypt_binlog=ON

‎.mariadb/no_encryption_key.key

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
1;dda0ccb18a28b0b4c2448b5f0217a134

‎README.md

+10-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ python-mysql-replication
44
<a href="https://travis-ci.org/noplay/python-mysql-replication"><img src="https://travis-ci.org/noplay/python-mysql-replication.svg?branch=master"></a>&nbsp;
55
<a href="https://pypi.python.org/pypi/mysql-replication"><img src="http://img.shields.io/pypi/dm/mysql-replication.svg"></a>
66

7-
Pure Python Implementation of MySQL replication protocol build on top of PyMYSQL. This allow you to receive event like insert, update, delete with their datas and raw SQL queries.
7+
Pure Python Implementation of MySQL replication protocol build on top of PyMYSQL. This allows you to receive event like insert, update, delete with their datas and raw SQL queries.
88

99
Use cases
1010
===========
@@ -56,6 +56,14 @@ Limitations
5656

5757
https://python-mysql-replication.readthedocs.org/en/latest/limitations.html
5858

59+
Featured
60+
=============
61+
62+
[Data Pipelines Pocket Reference](https://www.oreilly.com/library/view/data-pipelines-pocket/9781492087823/) (by James Densmore, O'Reilly): Introduced and exemplified in Chapter 4: Data Ingestion: Extracting Data.
63+
64+
[Streaming Changes in a Database with Amazon Kinesis](https://aws.amazon.com/blogs/database/streaming-changes-in-a-database-with-amazon-kinesis/) (by Emmanuel Espina, Amazon Web Services)
65+
66+
5967
Projects using this library
6068
===========================
6169

@@ -79,6 +87,7 @@ Projects using this library
7987
* MySQL to Kafka (https://github.com/scottpersinger/mysql-to-kafka/)
8088
* Aventri MySQL Monitor (https://github.com/aventri/mysql-monitor)
8189
* BitSwanPump: A real-time stream processor (https://github.com/LibertyAces/BitSwanPump)
90+
* clickhouse-mysql-data-reader: https://github.com/Altinity/clickhouse-mysql-data-reader
8291

8392
MySQL server settings
8493
=========================

‎docker-compose-test.yml

+93
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
version: '3.4'
2+
3+
x-mysql: &mysql
4+
environment:
5+
MYSQL_ALLOW_EMPTY_PASSWORD: true
6+
command: >
7+
mysqld
8+
--log-bin=mysql-bin.log
9+
--server-id 1
10+
--binlog-format=row
11+
--gtid_mode=on
12+
--enforce-gtid-consistency=on
13+
14+
x-mariadb: &mariadb
15+
environment:
16+
MARIADB_ALLOW_EMPTY_ROOT_PASSWORD: 1
17+
command: >
18+
--server-id=1
19+
--default-authentication-plugin=mysql_native_password
20+
--log-bin=master-bin
21+
--binlog-format=row
22+
23+
services:
24+
percona-5.7-ctl:
25+
<<: *mysql
26+
image: percona:5.7
27+
ports:
28+
- "3307:3306"
29+
networks:
30+
- default
31+
32+
percona-5.7:
33+
<<: *mysql
34+
image: percona:5.7
35+
ports:
36+
- "3306:3306"
37+
networks:
38+
- default
39+
40+
mariadb-10.6:
41+
<<: *mariadb
42+
image: mariadb:10.6
43+
ports:
44+
- "3308:3306"
45+
volumes:
46+
- type: bind
47+
source: ./.mariadb
48+
target: /opt/key_file
49+
- type: bind
50+
source: ./.mariadb/my.cnf
51+
target: /etc/mysql/my.cnf
52+
networks:
53+
- default
54+
55+
pymysqlreplication:
56+
build:
57+
context: .
58+
dockerfile: test.Dockerfile
59+
args:
60+
BASE_IMAGE: python:3.11-alpine
61+
MYSQL_5_7: percona-5.7
62+
MYSQL_5_7_CTL: percona-5.7-ctl
63+
MYSQL_5_7_CTL_PORT: 3306
64+
MARIADB_10_6: mariadb-10.6
65+
MARIADB_10_6_PORT: 3306
66+
67+
command:
68+
- /bin/sh
69+
- -ce
70+
- |
71+
echo "wait mysql server"
72+
73+
while :
74+
do
75+
if mysql -h percona-5.7 --user=root --execute "SELECT version();" 2>&1 >/dev/null && mysql -h percona-5.7-ctl --user=root --execute "SELECT version();" 2>&1 >/dev/null; then
76+
break
77+
fi
78+
sleep 1
79+
done
80+
81+
echo "run pytest"
82+
pytest -k "not test_no_trailing_rotate_event and not test_end_log_pos"
83+
84+
working_dir: /pymysqlreplication
85+
networks:
86+
- default
87+
depends_on:
88+
- percona-5.7
89+
- percona-5.7-ctl
90+
91+
networks:
92+
default:
93+
driver: bridge

‎docker-compose.yml

+39-9
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,47 @@
1-
version: '3.2'
1+
version: '3.4'
2+
3+
x-mysql: &mysql
4+
environment:
5+
MYSQL_ALLOW_EMPTY_PASSWORD: true
6+
command: >
7+
mysqld
8+
--log-bin=mysql-bin.log
9+
--server-id 1
10+
--binlog-format=row
11+
--gtid_mode=on
12+
--enforce-gtid-consistency=on
13+
14+
x-mariadb: &mariadb
15+
environment:
16+
MARIADB_ALLOW_EMPTY_ROOT_PASSWORD: 1
17+
command: >
18+
--log-bin=master-bin
19+
--server-id=1
20+
--default-authentication-plugin=mysql_native_password
21+
--binlog-format=row
22+
223
services:
324
percona-5.7:
25+
<<: *mysql
426
image: percona:5.7
5-
environment:
6-
MYSQL_ALLOW_EMPTY_PASSWORD: true
727
ports:
8-
- 3306:3306
9-
command: mysqld --log-bin=mysql-bin.log --server-id 1 --binlog-format=row --gtid_mode=on --enforce-gtid-consistency=on --log_slave_updates
28+
- "3306:3306"
1029

1130
percona-5.7-ctl:
31+
<<: *mysql
1232
image: percona:5.7
13-
environment:
14-
MYSQL_ALLOW_EMPTY_PASSWORD: true
1533
ports:
16-
- 3307:3307
17-
command: mysqld --log-bin=mysql-bin.log --server-id 1 --binlog-format=row --gtid_mode=on --enforce-gtid-consistency=on --log_slave_updates -P 3307
34+
- "3307:3306"
35+
36+
mariadb-10.6:
37+
<<: *mariadb
38+
image: mariadb:10.6
39+
ports:
40+
- "3308:3306"
41+
volumes:
42+
- type: bind
43+
source: ./.mariadb
44+
target: /opt/key_file
45+
- type: bind
46+
source: ./.mariadb/my.cnf
47+
target: /etc/mysql/my.cnf

‎docs/developement.rst

+1-1
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ When it's possible we have an unit test.
2323
*pymysqlreplication/tests/* contains the test suite. The test suite
2424
use the standard *unittest* Python module.
2525

26-
**Be carefull** tests will reset the binary log of your MySQL server.
26+
**Be careful** tests will reset the binary log of your MySQL server.
2727

2828
Make sure you have the following configuration set in your mysql config file (usually my.cnf on development env):
2929

‎examples/mariadb_gtid/read_event.py

+6-3
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import pymysql
22

33
from pymysqlreplication import BinLogStreamReader, gtid
4-
from pymysqlreplication.event import GtidEvent, RotateEvent, MariadbGtidEvent, QueryEvent
4+
from pymysqlreplication.event import GtidEvent, RotateEvent, MariadbGtidEvent, QueryEvent,MariadbAnnotateRowsEvent, MariadbBinLogCheckPointEvent
55
from pymysqlreplication.row_event import WriteRowsEvent, UpdateRowsEvent, DeleteRowsEvent
66

77
MARIADB_SETTINGS = {
@@ -62,13 +62,16 @@ def query_server_id(self):
6262
blocking=False,
6363
only_events=[
6464
MariadbGtidEvent,
65+
MariadbBinLogCheckPointEvent,
6566
RotateEvent,
6667
WriteRowsEvent,
6768
UpdateRowsEvent,
68-
DeleteRowsEvent
69+
DeleteRowsEvent,
70+
MariadbAnnotateRowsEvent
6971
],
7072
auto_position=gtid,
71-
is_mariadb=True
73+
is_mariadb=True,
74+
annotate_rows_event=True
7275
)
7376

7477
print('Starting reading events from GTID ', gtid)

0 commit comments

Comments
 (0)