From bb6d69f87577ac80f017635d23e4cfa3ff05764d Mon Sep 17 00:00:00 2001 From: Adam Mizerski Date: Wed, 5 Mar 2025 17:02:46 +0100 Subject: [PATCH] Improve tests --- .github/ISSUE_TEMPLATE.md | 4 +- .github/PULL_REQUEST_TEMPLATE.md | 1 - .github/workflows/coverage.yml | 4 +- .github/workflows/pytest.yml | 9 +- .pre-commit-config.yaml | 4 +- CHANGELOG | 2 +- docker-compose-test.yml | 43 +++--- examples/mysql_to_rabbitmq.py | 20 +-- getting-started.md | 1 - pymysqlreplication/tests/base.py | 23 +++- pymysqlreplication/tests/config.json | 24 ++-- pymysqlreplication/tests/test_basic.py | 137 +++++++++++++------- pymysqlreplication/tests/test_data_type.py | 104 ++++++++++----- pymysqlreplication/tests/test_event.py | 6 +- pymysqlreplication/tests/test_util_bytes.py | 2 +- pyproject.toml | 4 +- test.Dockerfile | 25 ++-- 17 files changed, 253 insertions(+), 160 deletions(-) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index b4173717f..c1d7bc0cd 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -3,8 +3,8 @@ Please specify the versions you are using. Exact version numbers are preferred. - Pymyrepl (e.g., 1.0.2): - OS (e.g., Ubuntu 18.04): - Database and version (Remove unnecessary options): - - MySQL: - - MariaDB: + - MySQL: + - MariaDB: - Percona: ### System Variables diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 005d44b83..7d71f4daf 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -18,4 +18,3 @@ ### Additional Information - diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 250ad7d66..4fe5897b3 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -1,5 +1,5 @@ name: Python Coverage -on: +on: push: pull_request: workflow_dispatch: @@ -7,7 +7,7 @@ jobs: build: runs-on: ubuntu-latest timeout-minutes: 3 - + steps: - name: Check out code uses: actions/checkout@v4 diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index fb1d32894..7347f2e12 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -16,7 +16,8 @@ jobs: - {name: 'Pypy 3.7', python: 'pypy-3.7'} - {name: 'Pypy 3.10', python: 'pypy-3.10'} name: ${{ matrix.name }} - runs-on: ubuntu-latest + # ubuntu-latest does not have python 3.7 + runs-on: ubuntu-22.04 timeout-minutes: 3 steps: @@ -53,14 +54,10 @@ jobs: working-directory: pymysqlreplication/tests run: pytest -k "$PYTEST_SKIP_OPTION" --db=mysql-5 - - name: Run tests for mysql-5-ctl - working-directory: pymysqlreplication/tests - run: pytest -k "$PYTEST_SKIP_OPTION" --db=mysql-5-ctl - - name: Run tests for mysql-8 working-directory: pymysqlreplication/tests run: pytest -k "$PYTEST_SKIP_OPTION" --db=mysql-8 - name: Run tests for mariadb-10 working-directory: pymysqlreplication/tests - run: pytest -k "$PYTEST_SKIP_OPTION" -m mariadb --db=mariadb-10 + run: pytest -k "$PYTEST_SKIP_OPTION" --db=mariadb-10 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4a2bf6771..53b74ec5c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v5.0.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer @@ -8,7 +8,7 @@ repos: - id: check-added-large-files - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.286 + rev: v0.9.10 hooks: - id: ruff args: [ --fix, --exit-non-zero-on-fix ] diff --git a/CHANGELOG b/CHANGELOG index 0c3f80059..4fe99e65f 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -259,4 +259,4 @@ 1.0.9 11/08/2024 * Fix typo in ident variable name (#619) -* Remove black and use only ruff as linter \ No newline at end of file +* Remove black and use only ruff as linter diff --git a/docker-compose-test.yml b/docker-compose-test.yml index 7495cba68..3a8866df8 100644 --- a/docker-compose-test.yml +++ b/docker-compose-test.yml @@ -1,5 +1,3 @@ -version: '3.4' - x-mysql: &mysql environment: MYSQL_ALLOW_EMPTY_PASSWORD: true @@ -21,19 +19,9 @@ x-mariadb: &mariadb --binlog-format=row services: - percona-5.7-ctl: - <<: *mysql - image: percona:5.7 - ports: - - "3307:3306" - networks: - - default - percona-5.7: <<: *mysql image: percona:5.7 - ports: - - "3306:3306" networks: - default @@ -41,16 +29,12 @@ services: <<: *mysql image: percona:8.0 platform: linux/amd64 - ports: - - "3309:3306" networks: - default mariadb-10.6: <<: *mariadb image: mariadb:10.6 - ports: - - "3308:3306" volumes: - type: bind source: ./.mariadb @@ -66,10 +50,7 @@ services: context: . dockerfile: test.Dockerfile args: - BASE_IMAGE: python:3.11-alpine MYSQL_5_7: percona-5.7 - MYSQL_5_7_CTL: percona-5.7-ctl - MYSQL_5_7_CTL_PORT: 3306 MYSQL_8_0: percona-8.0 MYSQL_8_0_PORT: 3306 MARIADB_10_6: mariadb-10.6 @@ -81,24 +62,34 @@ services: - | echo "wait mysql server" - while : - do - if mysql -h percona-5.7 --user=root --execute "SELECT version();" 2>&1 >/dev/null && mysql -h percona-5.7-ctl --user=root --execute "SELECT version();" 2>&1 >/dev/null && mysql -h percona-8.0 --user=root --execute "SELECT version();" 2>&1 >/dev/null; then - break - fi + check_servers_up() { + for s in percona-5.7 percona-8.0 mariadb-10.6; do + mariadb -h "$$s" --user=root --execute "SELECT version();" || { + echo "$$h not ready" + return 1 + } + done + return 0 + } + + while ! check_servers_up; do sleep 1 done echo "run pytest" - pytest -k "not test_no_trailing_rotate_event and not test_end_log_pos" + for db in mysql-5 mariadb-10 mysql-8; do + echo "testing with $$db" + pytest -vv -k "not test_no_trailing_rotate_event and not test_end_log_pos" --db "$$db" + echo "tested with $$db" + done working_dir: /pymysqlreplication networks: - default depends_on: - percona-5.7 - - percona-5.7-ctl - percona-8.0 + - mariadb-10.6 networks: default: diff --git a/examples/mysql_to_rabbitmq.py b/examples/mysql_to_rabbitmq.py index 8e6e527d8..e16f60601 100644 --- a/examples/mysql_to_rabbitmq.py +++ b/examples/mysql_to_rabbitmq.py @@ -16,13 +16,13 @@ WriteRowsEvent, ) -MYSQL_SETTINGS = {"host": "127.0.0.1", "port": 3306, "user": "root", +MYSQL_SETTINGS = {"host": "127.0.0.1", "port": 3306, "user": "root", "passwd": "password"} def main(): stream = BinLogStreamReader( connection_settings=MYSQL_SETTINGS, - server_id=3, + server_id=3, only_events=[DeleteRowsEvent, WriteRowsEvent, UpdateRowsEvent], ) @@ -31,14 +31,14 @@ def main(): password='password' ) params = pika.ConnectionParameters('rabbitmq_host', credentials=credentials) - + # RabbitMQ Connection Settings conn = pika.BlockingConnection(params) channel = conn.channel() channel.queue_declare(queue='order') channel.exchange_declare(durable=True, exchange_type='direct', exchange='direct') channel.queue_bind(queue='order', exchange='direct', routing_key='order') - + for binlogevent in stream: for row in binlogevent.rows: if isinstance(binlogevent, DeleteRowsEvent): @@ -48,12 +48,12 @@ def main(): elif isinstance(binlogevent, UpdateRowsEvent): routing_key = "order" message_body = row["after_values"].items() - + elif isinstance(binlogevent, WriteRowsEvent): routing_key = "order" message_body = row["values"].items() - - properties = pika.BasicProperties(content_type='application/json', + + properties = pika.BasicProperties(content_type='application/json', delivery_mode=DeliveryMode.Transient) channel.basic_publish( exchange='direct', @@ -61,10 +61,10 @@ def main(): body=json.dumps(message_body, default=lambda x: str(x)), properties=properties ) - + stream.close() conn.close() - + if __name__ == '__main__': - main() \ No newline at end of file + main() diff --git a/getting-started.md b/getting-started.md index 8b1378917..e69de29bb 100644 --- a/getting-started.md +++ b/getting-started.md @@ -1 +0,0 @@ - diff --git a/pymysqlreplication/tests/base.py b/pymysqlreplication/tests/base.py index b8988ce0d..b2d616590 100644 --- a/pymysqlreplication/tests/base.py +++ b/pymysqlreplication/tests/base.py @@ -45,8 +45,19 @@ def setUpDatabase(self, get_db): def setUp(self, charset="utf8"): # default self.conn_control = None + + for i in ['host', 'port']: + env_override = self.database.pop(f'{i}_env_override', None) + if env_override: + override = os.environ.get(env_override) + if override: + if i == 'port': + override = int(override) + self.database[i] = override + db = copy.copy(self.database) db["db"] = None + db["charset"] = charset self.connect_conn_control(db) self.execute("DROP DATABASE IF EXISTS pymysqlreplication_test") self.execute("CREATE DATABASE pymysqlreplication_test") @@ -55,8 +66,8 @@ def setUp(self, charset="utf8"): self.connect_conn_control(db) self.stream = None self.resetBinLog() - self.isMySQL56AndMore() self.__is_mariaDB = None + self.isMySQL56AndMore() def getMySQLVersion(self): """Return the MySQL version of the server @@ -65,20 +76,28 @@ def getMySQLVersion(self): return self.execute("SELECT VERSION()").fetchone()[0].split("-")[0] def isMySQL56AndMore(self): + if self.isMariaDB(): + return False version = float(self.getMySQLVersion().rsplit(".", 1)[0]) if version >= 5.6: return True return False def isMySQL57(self): + if self.isMariaDB(): + return False version = float(self.getMySQLVersion().rsplit(".", 1)[0]) return version == 5.7 def isMySQL80AndMore(self): + if self.isMariaDB(): + return False version = float(self.getMySQLVersion().rsplit(".", 1)[0]) return version >= 8.0 def isMySQL8014AndMore(self): + if self.isMariaDB(): + return False version = float(self.getMySQLVersion().rsplit(".", 1)[0]) version_detail = int(self.getMySQLVersion().rsplit(".", 1)[1]) if version > 8.0: @@ -86,6 +105,8 @@ def isMySQL8014AndMore(self): return version == 8.0 and version_detail >= 14 def isMySQL8016AndMore(self): + if self.isMariaDB(): + return False version = float(self.getMySQLVersion().rsplit(".", 1)[0]) version_detail = int(self.getMySQLVersion().rsplit(".", 1)[1]) if version > 8.0: diff --git a/pymysqlreplication/tests/config.json b/pymysqlreplication/tests/config.json index 5bfd93a48..aeb3f881a 100644 --- a/pymysqlreplication/tests/config.json +++ b/pymysqlreplication/tests/config.json @@ -5,17 +5,9 @@ "passwd": "", "port": 3306, "use_unicode": true, - "charset": "utf8", - "db": "pymysqlreplication_test" - }, - "mysql-5-ctl": { - "host": "localhost", - "user": "root", - "passwd": "", - "port": 3307, - "use_unicode": true, - "charset": "utf8", - "db": "pymysqlreplication_test" + "db": "pymysqlreplication_test", + "host_env_override": "MYSQL_5_7", + "port_env_override": "MYSQL_5_7_PORT" }, "mariadb-10": { "host": "localhost", @@ -23,8 +15,9 @@ "passwd": "", "port": 3308, "use_unicode": true, - "charset": "utf8", - "db": "pymysqlreplication_test" + "db": "pymysqlreplication_test", + "host_env_override": "MARIADB_10_6", + "port_env_override": "MARIADB_10_6_PORT" }, "mysql-8": { "host": "localhost", @@ -32,7 +25,8 @@ "passwd": "", "port": 3309, "use_unicode": true, - "charset": "utf8", - "db": "pymysqlreplication_test" + "db": "pymysqlreplication_test", + "host_env_override": "MYSQL_8_0", + "port_env_override": "MYSQL_8_0_PORT" } } diff --git a/pymysqlreplication/tests/test_basic.py b/pymysqlreplication/tests/test_basic.py index a0ad0a9cb..fc3b635a3 100644 --- a/pymysqlreplication/tests/test_basic.py +++ b/pymysqlreplication/tests/test_basic.py @@ -13,7 +13,6 @@ from pymysqlreplication.packet import BinLogPacketWrapper from pymysql.protocol import MysqlPacket from unittest.mock import patch -import pytest __all__ = [ @@ -32,7 +31,14 @@ class TestBasicBinLogStreamReader(base.PyMySQLReplicationTestCase): def ignoredEvents(self): - return [GtidEvent, PreviousGtidsEvent] + return [ + GtidEvent, + PreviousGtidsEvent, + MariadbStartEncryptionEvent, + MariadbGtidListEvent, + MariadbBinLogCheckPointEvent, + MariadbGtidEvent, + ] def test_allowed_event_list(self): self.assertEqual(len(self.stream._allowed_event_list(None, None, False)), 25) @@ -247,7 +253,8 @@ def test_write_row_event(self): # QueryEvent for the Create Table self.assertIsInstance(self.stream.fetchone(), QueryEvent) # QueryEvent for the BEGIN - self.assertIsInstance(self.stream.fetchone(), QueryEvent) + if not self.isMariaDB(): + self.assertIsInstance(self.stream.fetchone(), QueryEvent) self.assertIsInstance(self.stream.fetchone(), TableMapEvent) @@ -280,7 +287,8 @@ def test_delete_row_event(self): self.assertIsInstance(self.stream.fetchone(), FormatDescriptionEvent) # QueryEvent for the BEGIN - self.assertIsInstance(self.stream.fetchone(), QueryEvent) + if not self.isMariaDB(): + self.assertIsInstance(self.stream.fetchone(), QueryEvent) self.assertIsInstance(self.stream.fetchone(), TableMapEvent) @@ -310,7 +318,8 @@ def test_update_row_event(self): self.assertIsInstance(self.stream.fetchone(), FormatDescriptionEvent) # QueryEvent for the BEGIN - self.assertIsInstance(self.stream.fetchone(), QueryEvent) + if not self.isMariaDB(): + self.assertIsInstance(self.stream.fetchone(), QueryEvent) self.assertIsInstance(self.stream.fetchone(), TableMapEvent) @@ -340,7 +349,8 @@ def test_minimal_image_write_row_event(self): # QueryEvent for the Create Table self.assertIsInstance(self.stream.fetchone(), QueryEvent) # QueryEvent for the BEGIN - self.assertIsInstance(self.stream.fetchone(), QueryEvent) + if not self.isMariaDB(): + self.assertIsInstance(self.stream.fetchone(), QueryEvent) self.assertIsInstance(self.stream.fetchone(), TableMapEvent) @@ -374,7 +384,8 @@ def test_minimal_image_delete_row_event(self): self.assertIsInstance(self.stream.fetchone(), FormatDescriptionEvent) # QueryEvent for the BEGIN - self.assertIsInstance(self.stream.fetchone(), QueryEvent) + if not self.isMariaDB(): + self.assertIsInstance(self.stream.fetchone(), QueryEvent) self.assertIsInstance(self.stream.fetchone(), TableMapEvent) @@ -405,7 +416,8 @@ def test_minimal_image_update_row_event(self): self.assertIsInstance(self.stream.fetchone(), FormatDescriptionEvent) # QueryEvent for the BEGIN - self.assertIsInstance(self.stream.fetchone(), QueryEvent) + if not self.isMariaDB(): + self.assertIsInstance(self.stream.fetchone(), QueryEvent) self.assertIsInstance(self.stream.fetchone(), TableMapEvent) @@ -446,7 +458,8 @@ def test_default_charset_parsing(self): # QueryEvent for the Create Table self.assertIsInstance(self.stream.fetchone(), QueryEvent) # QueryEvent for the BEGIN - self.assertIsInstance(self.stream.fetchone(), QueryEvent) + if not self.isMariaDB(): + self.assertIsInstance(self.stream.fetchone(), QueryEvent) event = self.stream.fetchone() self.assertIsInstance(event, TableMapEvent) @@ -489,9 +502,11 @@ def test_log_pos(self): self.assertIsInstance(self.stream.fetchone(), RotateEvent) self.assertIsInstance(self.stream.fetchone(), FormatDescriptionEvent) - self.assertIsInstance(self.stream.fetchone(), XidEvent) + if not self.isMariaDB(): + self.assertIsInstance(self.stream.fetchone(), XidEvent) # QueryEvent for the BEGIN - self.assertIsInstance(self.stream.fetchone(), QueryEvent) + if not self.isMariaDB(): + self.assertIsInstance(self.stream.fetchone(), QueryEvent) self.assertIsInstance(self.stream.fetchone(), TableMapEvent) self.assertIsInstance(self.stream.fetchone(), UpdateRowsEvent) self.assertIsInstance(self.stream.fetchone(), XidEvent) @@ -521,7 +536,8 @@ def test_log_pos_handles_disconnects(self): self.assertGreater(self.stream.log_pos, 0) self.assertIsInstance(self.stream.fetchone(), QueryEvent) - self.assertIsInstance(self.stream.fetchone(), QueryEvent) + if not self.isMariaDB(): + self.assertIsInstance(self.stream.fetchone(), QueryEvent) self.assertIsInstance(self.stream.fetchone(), TableMapEvent) self.assertIsInstance(self.stream.fetchone(), WriteRowsEvent) @@ -644,9 +660,9 @@ def test_json_update(self): self.database, server_id=1024, only_events=[UpdateRowsEvent] ) create_query = ( - "CREATE TABLE setting_table( id SERIAL AUTO_INCREMENT, setting JSON);" + "CREATE TABLE setting_table( id INT, setting JSON);" ) - insert_query = """INSERT INTO setting_table (setting) VALUES ('{"btn": true, "model": false}');""" + insert_query = """INSERT INTO setting_table (id, setting) VALUES (1, '{"btn": true, "model": false}');""" update_query = """ UPDATE setting_table SET setting = JSON_REMOVE(setting, '$.model') @@ -666,6 +682,9 @@ def test_json_update(self): self.assertEqual(event.rows[0]["after_values"]["setting"], {b"btn": True}), def test_format_description_event(self): + if self.isMariaDB(): + self.skipTest("This is for MySQL. There is a second test_format_description_event for MariaDB") + self.stream.close() self.stream = BinLogStreamReader( self.database, @@ -700,7 +719,14 @@ def setUp(self): self.execute("SET GLOBAL binlog_row_image='FULL';") def ignoredEvents(self): - return [GtidEvent, PreviousGtidsEvent] + return [ + GtidEvent, + PreviousGtidsEvent, + MariadbStartEncryptionEvent, + MariadbGtidListEvent, + MariadbBinLogCheckPointEvent, + MariadbGtidEvent, + ] def test_insert_multiple_row_event(self): query = "CREATE TABLE test (id INT NOT NULL AUTO_INCREMENT, data VARCHAR (50) NOT NULL, PRIMARY KEY (id))" @@ -715,7 +741,8 @@ def test_insert_multiple_row_event(self): self.assertIsInstance(self.stream.fetchone(), RotateEvent) self.assertIsInstance(self.stream.fetchone(), FormatDescriptionEvent) # QueryEvent for the BEGIN - self.assertIsInstance(self.stream.fetchone(), QueryEvent) + if not self.isMariaDB(): + self.assertIsInstance(self.stream.fetchone(), QueryEvent) self.assertIsInstance(self.stream.fetchone(), TableMapEvent) @@ -750,7 +777,8 @@ def test_update_multiple_row_event(self): self.assertIsInstance(self.stream.fetchone(), RotateEvent) self.assertIsInstance(self.stream.fetchone(), FormatDescriptionEvent) # QueryEvent for the BEGIN - self.assertIsInstance(self.stream.fetchone(), QueryEvent) + if not self.isMariaDB(): + self.assertIsInstance(self.stream.fetchone(), QueryEvent) self.assertIsInstance(self.stream.fetchone(), TableMapEvent) @@ -790,7 +818,8 @@ def test_delete_multiple_row_event(self): self.assertIsInstance(self.stream.fetchone(), FormatDescriptionEvent) # QueryEvent for the BEGIN - self.assertIsInstance(self.stream.fetchone(), QueryEvent) + if not self.isMariaDB(): + self.assertIsInstance(self.stream.fetchone(), QueryEvent) self.assertIsInstance(self.stream.fetchone(), TableMapEvent) @@ -1144,6 +1173,12 @@ def setUp(self): ) self.execute("SET @@binlog_format='STATEMENT'") + self.charset_num = 33 + if self.isMariaDB(): + self.charset_num = 8 + # if self.isMySQL80AndMore(): + # self.charset_num = 255 + def test_rand_event(self): self.execute( "CREATE TABLE test (id INT NOT NULL AUTO_INCREMENT, data INT NOT NULL, PRIMARY KEY (id))" @@ -1153,7 +1188,8 @@ def test_rand_event(self): self.assertEqual(self.bin_log_format(), "STATEMENT") self.assertIsInstance(self.stream.fetchone(), QueryEvent) - self.assertIsInstance(self.stream.fetchone(), QueryEvent) + if not self.isMariaDB(): + self.assertIsInstance(self.stream.fetchone(), QueryEvent) expected_rand_event = self.stream.fetchone() self.assertIsInstance(expected_rand_event, RandEvent) @@ -1171,7 +1207,8 @@ def test_user_var_string_event(self, mock_stdout): self.assertEqual(self.bin_log_format(), "STATEMENT") self.assertIsInstance(self.stream.fetchone(), QueryEvent) - self.assertIsInstance(self.stream.fetchone(), QueryEvent) + if not self.isMariaDB(): + self.assertIsInstance(self.stream.fetchone(), QueryEvent) expected_user_var_event = self.stream.fetchone() self.assertIsInstance(expected_user_var_event, UserVarEvent) @@ -1180,7 +1217,9 @@ def test_user_var_string_event(self, mock_stdout): self.assertEqual(expected_user_var_event.value, "foo") self.assertEqual(expected_user_var_event.is_null, 0) self.assertEqual(expected_user_var_event.type, 0) - self.assertEqual(expected_user_var_event.charset, 33) + if self.isMariaDB(): + self.charset_num = 33 + self.assertEqual(expected_user_var_event.charset, self.charset_num) # Test _dump method expected_user_var_event._dump() @@ -1197,7 +1236,8 @@ def test_user_var_real_event(self): self.assertEqual(self.bin_log_format(), "STATEMENT") self.assertIsInstance(self.stream.fetchone(), QueryEvent) - self.assertIsInstance(self.stream.fetchone(), QueryEvent) + if not self.isMariaDB(): + self.assertIsInstance(self.stream.fetchone(), QueryEvent) expected_user_var_event = self.stream.fetchone() self.assertIsInstance(expected_user_var_event, UserVarEvent) @@ -1206,7 +1246,7 @@ def test_user_var_real_event(self): self.assertIsInstance(expected_user_var_event.value, float) self.assertEqual(expected_user_var_event.is_null, 0) self.assertEqual(expected_user_var_event.type, 1) - self.assertEqual(expected_user_var_event.charset, 33) + self.assertEqual(expected_user_var_event.charset, self.charset_num) def test_user_var_int_event(self): self.execute( @@ -1222,7 +1262,8 @@ def test_user_var_int_event(self): self.assertEqual(self.bin_log_format(), "STATEMENT") self.assertIsInstance(self.stream.fetchone(), QueryEvent) - self.assertIsInstance(self.stream.fetchone(), QueryEvent) + if not self.isMariaDB(): + self.assertIsInstance(self.stream.fetchone(), QueryEvent) expected_user_var_event = self.stream.fetchone() self.assertIsInstance(expected_user_var_event, UserVarEvent) @@ -1231,7 +1272,7 @@ def test_user_var_int_event(self): self.assertEqual(expected_user_var_event.value, 5) self.assertEqual(expected_user_var_event.is_null, 0) self.assertEqual(expected_user_var_event.type, 2) - self.assertEqual(expected_user_var_event.charset, 33) + self.assertEqual(expected_user_var_event.charset, self.charset_num) expected_user_var_event = self.stream.fetchone() self.assertIsInstance(expected_user_var_event, UserVarEvent) @@ -1240,7 +1281,7 @@ def test_user_var_int_event(self): self.assertEqual(expected_user_var_event.value, 0) self.assertEqual(expected_user_var_event.is_null, 0) self.assertEqual(expected_user_var_event.type, 2) - self.assertEqual(expected_user_var_event.charset, 33) + self.assertEqual(expected_user_var_event.charset, self.charset_num) expected_user_var_event = self.stream.fetchone() self.assertIsInstance(expected_user_var_event, UserVarEvent) @@ -1249,7 +1290,7 @@ def test_user_var_int_event(self): self.assertEqual(expected_user_var_event.value, -5) self.assertEqual(expected_user_var_event.is_null, 0) self.assertEqual(expected_user_var_event.type, 2) - self.assertEqual(expected_user_var_event.charset, 33) + self.assertEqual(expected_user_var_event.charset, self.charset_num) def test_user_var_int24_event(self): self.execute( @@ -1265,7 +1306,8 @@ def test_user_var_int24_event(self): self.assertEqual(self.bin_log_format(), "STATEMENT") self.assertIsInstance(self.stream.fetchone(), QueryEvent) - self.assertIsInstance(self.stream.fetchone(), QueryEvent) + if not self.isMariaDB(): + self.assertIsInstance(self.stream.fetchone(), QueryEvent) expected_user_var_event = self.stream.fetchone() self.assertIsInstance(expected_user_var_event, UserVarEvent) @@ -1274,7 +1316,7 @@ def test_user_var_int24_event(self): self.assertEqual(expected_user_var_event.value, 8388607) self.assertEqual(expected_user_var_event.is_null, 0) self.assertEqual(expected_user_var_event.type, 2) - self.assertEqual(expected_user_var_event.charset, 33) + self.assertEqual(expected_user_var_event.charset, self.charset_num) expected_user_var_event = self.stream.fetchone() self.assertIsInstance(expected_user_var_event, UserVarEvent) @@ -1283,7 +1325,7 @@ def test_user_var_int24_event(self): self.assertEqual(expected_user_var_event.value, -8388607) self.assertEqual(expected_user_var_event.is_null, 0) self.assertEqual(expected_user_var_event.type, 2) - self.assertEqual(expected_user_var_event.charset, 33) + self.assertEqual(expected_user_var_event.charset, self.charset_num) expected_user_var_event = self.stream.fetchone() self.assertIsInstance(expected_user_var_event, UserVarEvent) @@ -1292,7 +1334,7 @@ def test_user_var_int24_event(self): self.assertEqual(expected_user_var_event.value, 16777215) self.assertEqual(expected_user_var_event.is_null, 0) self.assertEqual(expected_user_var_event.type, 2) - self.assertEqual(expected_user_var_event.charset, 33) + self.assertEqual(expected_user_var_event.charset, self.charset_num) def test_user_var_longlong_event(self): self.execute( @@ -1308,7 +1350,8 @@ def test_user_var_longlong_event(self): self.assertEqual(self.bin_log_format(), "STATEMENT") self.assertIsInstance(self.stream.fetchone(), QueryEvent) - self.assertIsInstance(self.stream.fetchone(), QueryEvent) + if not self.isMariaDB(): + self.assertIsInstance(self.stream.fetchone(), QueryEvent) expected_user_var_event = self.stream.fetchone() self.assertIsInstance(expected_user_var_event, UserVarEvent) @@ -1317,7 +1360,7 @@ def test_user_var_longlong_event(self): self.assertEqual(expected_user_var_event.value, 9223372036854775807) self.assertEqual(expected_user_var_event.is_null, 0) self.assertEqual(expected_user_var_event.type, 2) - self.assertEqual(expected_user_var_event.charset, 33) + self.assertEqual(expected_user_var_event.charset, self.charset_num) expected_user_var_event = self.stream.fetchone() self.assertIsInstance(expected_user_var_event, UserVarEvent) @@ -1326,7 +1369,7 @@ def test_user_var_longlong_event(self): self.assertEqual(expected_user_var_event.value, -9223372036854775808) self.assertEqual(expected_user_var_event.is_null, 0) self.assertEqual(expected_user_var_event.type, 2) - self.assertEqual(expected_user_var_event.charset, 33) + self.assertEqual(expected_user_var_event.charset, self.charset_num) expected_user_var_event = self.stream.fetchone() self.assertIsInstance(expected_user_var_event, UserVarEvent) @@ -1335,7 +1378,7 @@ def test_user_var_longlong_event(self): self.assertEqual(expected_user_var_event.value, 18446744073709551615) self.assertEqual(expected_user_var_event.is_null, 0) self.assertEqual(expected_user_var_event.type, 2) - self.assertEqual(expected_user_var_event.charset, 33) + self.assertEqual(expected_user_var_event.charset, self.charset_num) def test_user_var_decimal_event(self): self.execute( @@ -1350,7 +1393,8 @@ def test_user_var_decimal_event(self): self.assertEqual(self.bin_log_format(), "STATEMENT") self.assertIsInstance(self.stream.fetchone(), QueryEvent) - self.assertIsInstance(self.stream.fetchone(), QueryEvent) + if not self.isMariaDB(): + self.assertIsInstance(self.stream.fetchone(), QueryEvent) expected_user_var_event = self.stream.fetchone() self.assertIsInstance(expected_user_var_event, UserVarEvent) @@ -1359,7 +1403,7 @@ def test_user_var_decimal_event(self): self.assertEqual(expected_user_var_event.value, 5.25) self.assertEqual(expected_user_var_event.is_null, 0) self.assertEqual(expected_user_var_event.type, 4) - self.assertEqual(expected_user_var_event.charset, 33) + self.assertEqual(expected_user_var_event.charset, self.charset_num) expected_user_var_event = self.stream.fetchone() self.assertIsInstance(expected_user_var_event, UserVarEvent) @@ -1368,7 +1412,7 @@ def test_user_var_decimal_event(self): self.assertEqual(expected_user_var_event.value, -5.25) self.assertEqual(expected_user_var_event.is_null, 0) self.assertEqual(expected_user_var_event.type, 4) - self.assertEqual(expected_user_var_event.charset, 33) + self.assertEqual(expected_user_var_event.charset, self.charset_num) def tearDown(self): self.execute("SET @@binlog_format='ROW'") @@ -1376,7 +1420,6 @@ def tearDown(self): super(TestStatementConnectionSetting, self).tearDown() -@pytest.mark.mariadb class TestMariadbBinlogStreamReader(base.PyMySQLReplicationTestCase): def setUp(self): super().setUp() @@ -1413,7 +1456,6 @@ def test_binlog_checkpoint_event(self): self.assertEqual(event.filename, self.bin_log_basename() + ".000001") -@pytest.mark.mariadb class TestMariadbBinlogStreamReader2(base.PyMySQLReplicationTestCase): def setUp(self): super().setUp() @@ -1519,6 +1561,9 @@ def test_gtid_list_event(self): self.assertEqual(event.gtid_list[0].gtid, "0-1-15") def test_format_description_event(self): + if not self.isMariaDB(): + self.skipTest("This is for MariaDB. There is a second test_format_description_event for MySQL.") + self.stream.close() self.stream = BinLogStreamReader( self.database, @@ -1546,10 +1591,14 @@ def test_format_description_event(self): class TestRowsQueryLogEvents(base.PyMySQLReplicationTestCase): def setUp(self): super(TestRowsQueryLogEvents, self).setUp() - self.execute("SET SESSION binlog_rows_query_log_events=1") + if self.isMariaDB(): + self.skipTest("skipping the entire class") + else: + self.execute("SET SESSION binlog_rows_query_log_events=1") def tearDown(self): - self.execute("SET SESSION binlog_rows_query_log_events=0") + if not self.isMariaDB(): + self.execute("SET SESSION binlog_rows_query_log_events=0") super(TestRowsQueryLogEvents, self).tearDown() def test_rows_query_log_event(self): @@ -1618,7 +1667,6 @@ def test_query_event_latin1(self): assert event.query == r"CREATE TABLE test_latin1_\xd6\xc6\xdb (a INT)" -@pytest.mark.mariadb class TestOptionalMetaData(base.PyMySQLReplicationTestCase): def setUp(self): super(TestOptionalMetaData, self).setUp() @@ -1979,7 +2027,8 @@ def setUp(self): if not self.isMySQL8014AndMore(): self.skipTest("Mysql version is under 8.0.14 - pass TestJsonPartialUpdate") self.execute("SET SESSION binlog_row_image = 'FULL';") - self.execute("SET SESSION binlog_row_value_options = 'PARTIAL_JSON';") + if not self.isMariaDB(): + self.execute("SET SESSION binlog_row_value_options = 'PARTIAL_JSON';") def test_json_partial_update(self): create_query = "CREATE TABLE test_json_v2 (id INT, c JSON,PRIMARY KEY (id)) ;" diff --git a/pymysqlreplication/tests/test_data_type.py b/pymysqlreplication/tests/test_data_type.py index 4c03ab1cf..1f0e79245 100644 --- a/pymysqlreplication/tests/test_data_type.py +++ b/pymysqlreplication/tests/test_data_type.py @@ -13,26 +13,37 @@ __all__ = ["TestDataType", "TestDataTypeVersion8"] +def encode_value(v): + if isinstance(v, str): + return v.encode() + if isinstance(v, list): + return [encode_value(x) for x in v] + return v + + def to_binary_dict(d): - def encode_value(v): - if isinstance(v, str): - return v.encode() - if isinstance(v, list): - return [encode_value(x) for x in v] - return v + return {k.encode(): encode_value(v) for (k, v) in d.items()} - return dict([(k.encode(), encode_value(v)) for (k, v) in d.items()]) +def to_binary_list(lst): + return [encode_value(v) for v in lst] class TestDataType(base.PyMySQLReplicationTestCase): def setUp(self): super(TestDataType, self).setUp() - if self.isMySQL8014AndMore(): + if self.isMySQL8014AndMore() or self.isMariaDB(): self.execute("SET GLOBAL binlog_row_metadata='FULL';") self.execute("SET GLOBAL binlog_row_image='FULL';") def ignoredEvents(self): - return [GtidEvent, PreviousGtidsEvent] + return [ + GtidEvent, + PreviousGtidsEvent, + MariadbStartEncryptionEvent, + MariadbGtidListEvent, + MariadbBinLogCheckPointEvent, + MariadbGtidEvent, + ] def create_and_insert_value(self, create_query, insert_query): self.execute(create_query) @@ -45,7 +56,8 @@ def create_and_insert_value(self, create_query, insert_query): self.assertIsInstance(self.stream.fetchone(), QueryEvent) # QueryEvent for the BEGIN - self.assertIsInstance(self.stream.fetchone(), QueryEvent) + if not self.isMariaDB(): + self.assertIsInstance(self.stream.fetchone(), QueryEvent) self.assertIsInstance(self.stream.fetchone(), TableMapEvent) @@ -591,10 +603,12 @@ def test_json(self): insert_query = """INSERT INTO test (id, value) VALUES (1, '{"my_key": "my_val", "my_key2": "my_val2"}');""" event = self.create_and_insert_value(create_query, insert_query) if event.table_map[event.table_id].column_name_flag: - self.assertEqual( - event.rows[0]["values"]["value"], - {b"my_key": b"my_val", b"my_key2": b"my_val2"}, - ) + expected_value = {"my_key": "my_val", "my_key2": "my_val2"} + if self.isMariaDB(): + expected_value = json.dumps(expected_value) + else: + expected_value = to_binary_dict(expected_value) + self.assertEqual(event.rows[0]["values"]["value"], expected_value) def test_json_array(self): create_query = "CREATE TABLE test (id int, value json);" @@ -603,9 +617,16 @@ def test_json_array(self): ) event = self.create_and_insert_value(create_query, insert_query) if event.table_map[event.table_id].column_name_flag: - self.assertEqual(event.rows[0]["values"]["value"], [b"my_val", b"my_val2"]) + expected_value = ["my_val", "my_val2"] + if self.isMariaDB(): + expected_value = json.dumps(expected_value) + else: + expected_value = to_binary_list(expected_value) + self.assertEqual(event.rows[0]["values"]["value"], expected_value) def test_json_large(self): + if self.isMariaDB(): + self.skipTest("TODO: Fails on MariaDB") data = dict( [("foooo%i" % i, "baaaaar%i" % i) for i in range(2560)] ) # Make it large enough to reach 2^16 length @@ -619,6 +640,8 @@ def test_json_large(self): def test_json_large_array(self): "Test json array larger than 64k bytes" + if self.isMariaDB(): + self.skipTest("TODO: Fails on MariaDB") create_query = "CREATE TABLE test (id int, value json);" large_array = dict(my_key=[i for i in range(100000)]) insert_query = "INSERT INTO test (id, value) VALUES (1, '%s');" % ( @@ -631,6 +654,8 @@ def test_json_large_array(self): ) def test_json_large_with_literal(self): + if self.isMariaDB(): + self.skipTest("TODO: Fails on MariaDB") data = dict( [("foooo%i" % i, "baaaaar%i" % i) for i in range(2560)], literal=True ) # Make it large with literal @@ -665,7 +690,11 @@ def test_json_types(self): ) event = self.create_and_insert_value(create_query, insert_query) if event.table_map[event.table_id].column_name_flag: - self.assertEqual(event.rows[0]["values"]["value"], to_binary_dict(data)) + if self.isMariaDB(): + expected_value = json.dumps(data) + else: + expected_value = to_binary_dict(data) + self.assertEqual(event.rows[0]["values"]["value"], expected_value) self.tearDown() self.setUp() @@ -691,21 +720,33 @@ def test_json_basic(self): ) event = self.create_and_insert_value(create_query, insert_query) if event.table_map[event.table_id].column_name_flag: - self.assertEqual(event.rows[0]["values"]["value"], data) + expected_value = data + if self.isMariaDB(): + expected_value = json.dumps(expected_value) + self.assertEqual(event.rows[0]["values"]["value"], expected_value) self.tearDown() self.setUp() def test_json_unicode(self): + if self.isMariaDB(): + self.skipTest("TODO: Fails on MariaDB") + create_query = "CREATE TABLE test (id int, value json);" insert_query = """INSERT INTO test (id, value) VALUES (1, '{"miam": "🍔"}');""" event = self.create_and_insert_value(create_query, insert_query) if event.table_map[event.table_id].column_name_flag: - self.assertEqual( - event.rows[0]["values"]["value"][b"miam"], "🍔".encode("utf8") - ) + expected_value = {"miam": "🍔"} + if self.isMariaDB(): + expected_value = json.dumps(expected_value) + else: + expected_value = to_binary_dict(expected_value) + self.assertEqual(event.rows[0]["values"]["value"], expected_value) def test_json_long_string(self): + if self.isMariaDB(): + self.skipTest("TODO: Fails on MariaDB") + create_query = "CREATE TABLE test (id int, value json);" # The string length needs to be larger than what can fit in a single byte. string_value = "super_long_string" * 100 @@ -720,7 +761,9 @@ def test_json_long_string(self): to_binary_dict({"my_key": string_value}), ) - def test_json_deciaml_time_datetime(self): + def test_json_decimal_time_datetime(self): + if self.isMariaDB(): + self.skipTest("TODO: Fails on MariaDB") create_query = """CREATE TABLE json_deciaml_time_datetime_test ( id INT PRIMARY KEY AUTO_INCREMENT, json_data JSON @@ -729,14 +772,12 @@ def test_json_deciaml_time_datetime(self): INSERT INTO json_deciaml_time_datetime_test (json_data) VALUES (JSON_OBJECT('time_key', CAST('18:54:12' AS TIME), 'datetime_key', CAST('2023-09-24 18:54:12' AS DATETIME) ,'decimal', CAST('99.99' AS DECIMAL(10, 2))));""" event = self.create_and_insert_value(create_query, insert_query) if event.table_map[event.table_id].column_name_flag: - self.assertEqual( - event.rows[0]["values"]["json_data"], - { - b"decimal": Decimal("99.99"), - b"time_key": datetime.time(18, 54, 12), - b"datetime_key": datetime.datetime(2023, 9, 24, 18, 54, 12), - }, - ) + expected_data = { + b"decimal": Decimal("99.99"), + b"time_key": datetime.time(18, 54, 12), + b"datetime_key": datetime.datetime(2023, 9, 24, 18, 54, 12), + } + self.assertEqual(event.rows[0]["values"]["json_data"], expected_data) def test_null(self): create_query = "CREATE TABLE test ( \ @@ -824,6 +865,8 @@ def test_status_vars(self): Raises: AssertionError: if status variables not set correctly """ + if self.isMariaDB(): + self.skipTest("Not supported on MariaDB") create_query = "CREATE TABLE test (id INTEGER)" event = self.create_table(create_query) self.assertEqual(event.catalog_nz_code, b"std") @@ -884,7 +927,8 @@ def test_null_bitmask(self): self.assertIsInstance(self.stream.fetchone(), QueryEvent) # QueryEvent for the BEGIN - self.assertIsInstance(self.stream.fetchone(), QueryEvent) + if not self.isMariaDB(): + self.assertIsInstance(self.stream.fetchone(), QueryEvent) event = self.stream.fetchone() diff --git a/pymysqlreplication/tests/test_event.py b/pymysqlreplication/tests/test_event.py index 7d8a4f154..00529288e 100644 --- a/pymysqlreplication/tests/test_event.py +++ b/pymysqlreplication/tests/test_event.py @@ -6,10 +6,12 @@ class BinLogEventTestCase(PyMySQLReplicationTestCase): def setUp(self): super(BinLogEventTestCase, self).setUp() - self.execute("SET SESSION binlog_rows_query_log_events=1") + if not self.isMariaDB(): + self.execute("SET SESSION binlog_rows_query_log_events=1") def tearDown(self): - self.execute("SET SESSION binlog_rows_query_log_events=0") + if not self.isMariaDB(): + self.execute("SET SESSION binlog_rows_query_log_events=0") super(BinLogEventTestCase, self).tearDown() target_fields = ["timestamp", "log_pos", "event_size", "read_bytes"] diff --git a/pymysqlreplication/tests/test_util_bytes.py b/pymysqlreplication/tests/test_util_bytes.py index 8bb7afaa3..270f6f145 100644 --- a/pymysqlreplication/tests/test_util_bytes.py +++ b/pymysqlreplication/tests/test_util_bytes.py @@ -3,7 +3,7 @@ """ These tests aim to cover some potential input scenarios, including valid inputs, edge cases, and error handling. -This approach ensures that every line of the corresponding function is executed under test conditions, +This approach ensures that every line of the corresponding function is executed under test conditions, thereby improving the coverage. """ diff --git a/pyproject.toml b/pyproject.toml index 638c65c6c..879e1c805 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ -[tool.ruff] -lint.ignore = [ +[tool.ruff.lint] +ignore = [ "F403", # from module import *' used, It should be removed afterwad "F405", # same to F403 ] diff --git a/test.Dockerfile b/test.Dockerfile index 43d652a7a..e4c404cbb 100644 --- a/test.Dockerfile +++ b/test.Dockerfile @@ -1,23 +1,13 @@ -ARG BASE_IMAGE=${BASE_IMAGE:-python:3.11-alpine} -FROM ${BASE_IMAGE} +FROM python:3.11 -COPY .mariadb .mariadb -COPY pymysqlreplication pymysqlreplication -COPY README.md README.md -COPY setup.py setup.py -RUN apk add bind-tools -RUN apk add mysql-client -RUN pip install . +RUN apt-get update && apt-get install -y mariadb-client && rm -rf /var/lib/apt/lists/* RUN pip install pytest ARG MYSQL_5_7 ENV MYSQL_5_7 ${MYSQL_5_7} -ARG MYSQL_5_7_CTL -ENV MYSQL_5_7_CTL ${MYSQL_5_7_CTL} - -ARG MYSQL_5_7_CTL_PORT -ENV MYSQL_5_7_CTL_PORT ${MYSQL_5_7_CTL_PORT} +ARG MYSQL_5_7_PORT +ENV MYSQL_5_7_PORT ${MYSQL_5_7_PORT} ARG MYSQL_8_0 ENV MYSQL_8_0 ${MYSQL_8_0} @@ -30,3 +20,10 @@ ENV MARIADB_10_6 ${MARIADB_10_6} ARG MARIADB_10_6_PORT ENV MARIADB_10_6_PORT ${MARIADB_10_6_PORT} + +COPY .mariadb .mariadb +COPY README.md README.md +COPY setup.py setup.py +COPY pymysqlreplication pymysqlreplication + +RUN pip install .