packaging

Author: sunsingerus

README.md

@@ -64,7 +64,8 @@ sudo yum install ./build/bdist.linux-x86_64/rpm/RPMS/noarch/clickhouse-mysql-0.0
 ## RPM Installation
 **Tested on CentOS 7**
 
-Packagecloud repo
+Packagecloud repo from [packagecloud.io](https://packagecloud.io/Altinity/clickhouse)
+More details on installation are available on [https://github.com/Altinity/clickhouse-rpm-install](https://github.com/Altinity/clickhouse-rpm-install)
 ```bash
 curl -s https://packagecloud.io/install/repositories/altinity/clickhouse/script.rpm.sh | sudo bash
 ```
@@ -93,30 +94,42 @@ sudo /etc/init.d/clickhouse-mysql start
 In case you need just to use the app - this is the most convenient way to go.
 
 Install dependencies.
-MySQL repo for `mysql-community-devel`
+MySQL repo (for `mysql-community-devel`)
 ```bash
 sudo yum install -y https://dev.mysql.com/get/mysql57-community-release-el7-11.noarch.rpm
 ```
-epel (for python3)
+epel (for `python3`)
 ```bash
-sudo yum install epel-release
+sudo yum install -y epel-release
+```
+
+clickhouse-client (for `clickhouse-client`) from Packagecloud repo from [packagecloud.io](https://packagecloud.io/Altinity/clickhouse)
+More details on installation are available on [https://github.com/Altinity/clickhouse-rpm-install](https://github.com/Altinity/clickhouse-rpm-install)
+```bash
+curl -s https://packagecloud.io/install/repositories/altinity/clickhouse/script.rpm.sh | sudo bash
+```
+```bash
+sudo yum install -y clickhouse-client
 ```
+
 and direct dependencies:
 ```bash
-sudo yum install mysql-community-devel
-sudo yum install gcc
-sudo yum install python34-devel python34-pip
+sudo yum install -y mysql-community-devel
+sudo yum install -y mariadb-devel
+sudo yum install -y gcc
+sudo yum install -y python34-devel python34-pip
 ```
 
 Install datareader
 ```bash
 sudo pip3 install clickhouse-mysql
 ```
 
-Now we are able to call datareader as an app
+Now we are able to call datareader as an app - perform last installation steps - install service files, etc
 ```bash
 [user@localhost ~]$ which clickhouse-mysql
 /usr/bin/clickhouse-mysql
+/usr/bin/clickhouse-mysql --install
 ```
 
 ## GitHub-based Installation - Clone Sources

clickhouse-mysql.conf

@@ -0,0 +1 @@
+clickhouse_mysql.etc/clickhouse-mysql.conf

clickhouse-mysql.conf

@@ -0,0 +1,78 @@
+# INTRODUCTION
+#
+# 1. Parameters accept both variants for naming - with '_' as a words separator or '-'.
+# The former is used for ENV VARS, the latter - for CLI options.
+# Config file accepts both, thus 'log_file' equals to 'log-file'
+#
+# 2. Boolean parameters accept case-insensitive 'yes', 'no', 'on', 'off', 1 or 0 values
+#
+# 3. Each parameter must have a value, thus
+#    'dry=yes' is ok,
+#    'dry=' is not that good, but will be interpreted as an empty line, while
+#    'dry' is a syntax error
+#
+# 4. Comments are '#'-only. So, php-like ';' is not accepted as a comment symbol in config file
+
+#
+# general app section
+#
+
+#config_file=clickhouse-mysql.conf
+#log_file=/var/log/clickhouse-mysql/main.log
+log_level=debug
+nice_pause=1
+#dry=yes
+#daemon=yes
+#pid_file=/tmp/clickhouse-client.pid
+binlog_position_file=/tmp/clickhouse-mysql-binlog.pos
+#mempool=yes
+mempool_max_events_num=10000
+#mempool_max_rows_num=
+mempool_max_flush_interval=60
+csvpool=yes
+csvpool_file_path_prefix=qwe_
+#csvpool_keep_files=yes
+#table_templates=yes
+#table_templates_with_create_database=yes
+#table_templates_json=yes
+#table_migrate=yes
+
+#
+# src section
+#
+
+src_server_id=1
+src_host=127.0.0.1
+#src_port=3306
+src_user=reader
+src_password=qwerty
+#src_schemas=db1, db2, db3
+#src_tables=db1.t4 t5
+#src_tables_where_clauses=a=1
+src_tables_prefixes=log_, log1_, log2_
+src_wait=yes
+src_resume=yes
+#src_file=
+#src_binlog_file=mysql-bin.000024
+#src_binlog_position=5307
+
+#
+# dst section
+#
+
+#dst_file=dst.csv
+dst_host=127.0.0.1
+#dst_port=
+#dst_user=default
+#dst_password=
+#dst_schema=db
+dst_table=logunified
+
+#
+# converters section
+#
+
+#column_default_value=
+#column_skip=
+#ch_converter_file=
+#ch_converter_class=

clickhouse_mysql.etc/clickhouse-mysql.conf

@@ -13,8 +13,11 @@
 #SERVICE_NAME=$(basename $0)
 SERVICE_NAME=clickhouse-mysql
 
-PID_FILE=/var/run/${SERVICE_NAME}/${SERVICE_NAME}.pid
-LOG_FILE=/var/log/${SERVICE_NAME}/${SERVICE_NAME}.log
+PID_FILE_DIR="/var/run/${SERVICE_NAME}"
+LOG_FILE_DIR="/var/log/${SERVICE_NAME}"
+
+PID_FILE="${PID_FILE_DIR}/${SERVICE_NAME}.pid"
+LOG_FILE="${LOG_FILE_DIR}/${SERVICE_NAME}.log"
 
 #RUN_CMD="python3 -m clickhouse_mysql.main --config-file=clickhouse-mysql.conf --daemon --log-file=$LOG_FILE --pid-file=$PID_FILE"
 RUN_CMD="/usr/bin/clickhouse-mysql --daemon --log-file=$LOG_FILE --pid-file=$PID_FILE"
@@ -79,6 +82,19 @@ function start_incapable_service()
 	sudo -u $RUN_CMD_AS_USER bash -c "$CMD"
 }
 
+function create_dirs()
+{
+	if [ ! -d "${PID_FILE_DIR}" ]; then
+		mkdir -p "${PID_FILE_DIR}"
+		chown "${RUN_CMD_AS_USER}" "${PID_FILE_DIR}"
+	fi
+
+	if [ ! -d "${LOG_FILE_DIR}" ]; then
+		mkdir -p "${LOG_FILE_DIR}"
+		chown "${RUN_CMD_AS_USER}" "${LOG_FILE_DIR}"
+	fi
+}
+
 function start()
 {
 	if is_running; then
@@ -88,6 +104,8 @@ function start()
 		exit 1
 	fi
 
+	create_dirs
+
 	if ! is_file_usable "$LOG_FILE"; then
 		echo "Log file $LOG_FILE is not writable by user $RUN_CMD_AS_USER. Please check path"
 		exit 1

init.d/clickhouse-mysql renamed to clickhouse_mysql.init.d/clickhouse-mysql

@@ -208,6 +208,11 @@ def options():
             action='store_true',
             help='Migrate table(s). IMPORTANT!. Target table has to be created in ClickHouse already! See --table-templates option(s) for help.'
         )
+        argparser.add_argument(
+            '--install',
+            action='store_true',
+            help='Install service file(s)'
+        )
 
         #
         # src section
@@ -400,6 +405,7 @@ def options():
             'table_templates_with_create_database': args.table_templates_with_create_database,
             'table_templates_json': args.table_templates_json,
             'table_migrate': args.table_migrate,
+            'install': args.install,
 
             #
             # src section

init.d/clickhouse-mysql.service renamed to clickhouse_mysql.init.d/clickhouse-mysql.service

@@ -77,6 +77,7 @@ def __init__(self):
                 'mempool_max_rows_num': self.options['mempool_max_rows_num'],
                 'mempool_max_flush_interval': self.options['mempool_max_flush_interval'],
                 'csvpool': self.options.get_bool('csvpool'),
+                'install': self.options.get_bool('install'),
             },
 
             #
@@ -217,6 +218,9 @@ def is_table_templates_with_create_database(self):
     def is_table_templates_json(self):
         return self.config['app']['table_templates_json']
 
+    def is_install(self):
+        return self.config['app']['install']
+
     def table_builder(self):
         return TableBuilder(
             host=self.config['table_builder']['mysql']['host'],

clickhouse_mysql/clioptions.py

@@ -8,6 +8,8 @@
 import pprint
 import json
 import os
+import pkg_resources
+import shutil
 
 if sys.version_info < (3, 4):
     print("Python version is NOT OK, need 3.4 at least")
@@ -51,11 +53,55 @@ def __init__(self):
         logging.info(pprint.pformat(sys.path))
 #        mp.set_start_method('forkserver')
 
+    @staticmethod
+    def install():
+        # install service file
+        src_service_filepath = os.path.abspath(
+            pkg_resources.resource_filename('clickhouse_mysql', '../clickhouse_mysql.init.d/clickhouse-mysql'))
+        dst_service_dir = '/etc/init.d/'
+        dst_service_filepath = dst_service_dir + 'clickhouse-mysql'
+        try:
+            print("Install service file ", src_service_filepath, ' to ', dst_service_filepath)
+            # os.makedirs wants '/' at the end of the folder name
+            os.makedirs(name=dst_service_dir, exist_ok=True)
+            shutil.copy2(src_service_filepath, dst_service_filepath)
+        except Exception as e:
+            print(e)
+            print('Looks like you have no permissions to write to ', dst_service_filepath)
+            sys.exit(1)
+
+        # install config example file
+        src_service_filepath = os.path.abspath(
+            pkg_resources.resource_filename('clickhouse_mysql', '../clickhouse_mysql.etc/clickhouse-mysql.conf'))
+        dst_service_dir = '/etc/clickhouse-mysql/'
+        dst_service_filepath = dst_service_dir + 'clickhouse-mysql-example.conf'
+        try:
+            print("Install config file ", src_service_filepath, ' to ', dst_service_filepath)
+            # os.makedirs wants '/' at the end of the folder name
+            os.makedirs(name=dst_service_dir, exist_ok=True)
+            shutil.copy2(src_service_filepath, dst_service_filepath)
+        except Exception as e:
+            print(e)
+            print('Looks like you have no permissions to write to ', dst_service_filepath)
+            sys.exit(1)
+
+        print("Ensure clickhouse user exists")
+        os.system("useradd clickhouse")
+
+        print("Ensure var/run folders available")
+        os.system("mkdir -p /var/log/clickhouse-mysql")
+        os.system("chown clickhouse /var/log/clickhouse-mysql")
+        os.system("mkdir -p /var/run/clickhouse-mysql")
+        os.system("chown clickhouse /var/run/clickhouse-mysql")
+
     def run(self):
         try:
             # what action are we going to do
 
-            if self.config.is_table_templates():
+            if self.config.is_install():
+                Main.install()
+
+            elif self.config.is_table_templates():
                 # we are going to prepare table templates
                 templates = self.config.table_builder().templates(self.config.is_table_templates_json())