dev: review UTF8

Author: sunsingerus

clickhouse_mysql/dbclient/mysqlclient.py

@@ -63,6 +63,7 @@ def connect(self, db):
                 db=db,
                 cursorclass=self.cursorclass,
                 charset='utf8',
+                use_unicode=True,
             )
             self.cursor = self.connection.cursor()
             logging.debug("Connect to the database host={} user={} password={} db={}".format(

clickhouse_mysql/tablemigrator.py

@@ -183,9 +183,7 @@ def migrate_one_table_data(self, db=None, table=None):
 
         # build SQL statement
         full_table_name = self.create_full_table_name(db=db, table=table)
-        sql = "SELECT {0} FROM {1}".format(
-            ",".join(self.get_columns(db,full_table_name))
-            ,full_table_name)
+        sql = "SELECT {0} FROM {1}".format(",".join(self.get_columns(db, full_table_name)), full_table_name)
         # in case we have WHERE clause for this db.table - add it to SQL
         if db in self.where_clauses and table in self.where_clauses[db]:
             sql += " WHERE {}".format(self.where_clauses[db][table])

dev_run_cli_options_local_table_migration.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+# This script performs migration of one table test.books from local MySQL
+# into one table test.books in local ClickHouse
+# Tables are created manually by user and are expected by migrator to be in place
+# Migrator exists after all data from migrated table is copied into ClickHouse
+
+# ugly stub to suppress unsufficient sockets
+#sudo bash -c "echo 1 > /proc/sys/net/ipv4/tcp_tw_reuse"
+
+# run data reader with specified Python version
+
+PYTHON="python3"
+
+CH_MYSQL="-m clickhouse_mysql.main"
+
+if [ ! -d "clickhouse_mysql" ]; then
+    # no clickhouse_mysql dir available - step out of examples dir
+    cd ..
+fi
+
+$PYTHON $CH_MYSQL ${*:1} \
+    --src-server-id=1 \
+    --nice-pause=1 \
+    --log-level=debug \
+    --src-host=127.0.0.1 \
+    --src-user=reader \
+    --src-password=qwerty \
+    --src-tables=test.books \
+    --dst-host=127.0.0.1 \
+    --dst-schema=test \
+    --dst-table=books \
+    --csvpool \
+    --csvpool-file-path-prefix=qwe_ \
+    --mempool-max-flush-interval=60 \
+    --mempool-max-events-num=10000 \
+    --pump-data \
+    --migrate-table
+
+#    --log-file=ontime.log \
+#	--mempool
+#   --mempool-max-events-num=3
+#   --mempool-max-flush-interval=30
+#	--dst-file=dst.csv
+#	--dst-schema=db
+#   --dst-table=datatypes
+#	--csvpool-keep-files
+#    --log-level=info \

setup.py

@@ -7,7 +7,7 @@
     name="clickhouse-mysql",
 
     # version should comply with PEP440
-    version='0.0.20180321',
+    version='0.0.20190821',
 
     description='MySQL to ClickHouse data migrator',
     long_description='MySQL to ClickHouse data migrator',