Added tb config

Author: alexon1234

.gitignore

@@ -46,4 +46,5 @@ _build
 
 # Tinibird
 bl-*
-out-*
+out-*
+.e

clickhouse_mysql/clioptions.py

@@ -93,6 +93,10 @@ class CLIOptions(Options):
         #
         # general app section
         #
+
+        'tb_host': 'https://ui.tinybird.co',
+        'tb_token': None,
+
         'config_file': '/etc/clickhouse-mysql/clickhouse-mysql.conf',
         'log_file': None,
         'log_level': None,
@@ -171,6 +175,20 @@ def options(self):
         #
         # general app section
         #
+        argparser.add_argument(
+            '--tb-host',
+            type=str,
+            default=self.default_options['tb_host'],
+            help='Tinybird host'
+        )
+
+        argparser.add_argument(
+            '--tb-token',
+            type=str,
+            default=self.default_options['tb_token'],
+            help='Tinybird host'
+        )
+
         argparser.add_argument(
             '--config-file',
             type=str,
@@ -508,6 +526,11 @@ def options(self):
             #
             # general app section
             #
+
+            'tb_host': args.tb_host,
+            'tb_token': args.tb_token,
+
+
             'config_file': args.config_file,
             'log_file': args.log_file,
             'log_level': args.log_level,

clickhouse_mysql/config.py

@@ -6,7 +6,7 @@
 
 from clickhouse_mysql.writer.chwriter import CHWriter
 from clickhouse_mysql.writer.csvwriter import CSVWriter
-from clickhouse_mysql.writer.chcsvwriter import CHCSVWriter
+from clickhouse_mysql.writer.tbcsvwriter import TBCSVWriter
 from clickhouse_mysql.writer.poolwriter import PoolWriter
 from clickhouse_mysql.writer.processwriter import ProcessWriter
 from clickhouse_mysql.objectbuilder import ObjectBuilder
@@ -61,6 +61,10 @@ def __init__(self):
             #
             #
             #
+            'tinybird': {
+                'host': self.options['tb_host'],
+                'token': self.options['tb_token'],
+            },
             'app': {
                 'config_file': self.options['config_file'],
                 'log_file': self.options['log_file'],
@@ -359,8 +363,11 @@ def writer_builder_csvpool(self):
                 'dst_table': self.config['writer']['file']['dst_table'],
                 'dst_table_prefix': self.config['writer']['file']['dst_table_prefix'],
                 'next_writer_builder': ObjectBuilder(
-                    class_name=CHCSVWriter,
-                    constructor_params=self.config['writer']['clickhouse']
+                    class_name=TBCSVWriter,
+                    constructor_params={
+                        'tb_host': self.config['tinybird']['host'],
+                        'tb_token': self.config['tinybird']['token']
+                    }
                 ),
                 'converter_builder': self.converter_builder(CONVERTER_CSV),
             })

clickhouse_mysql/writer/chcsvwriter.py renamed to clickhouse_mysql/writer/tbcsvwriter.py

@@ -12,37 +12,39 @@
 from requests_toolbelt.multipart.encoder import MultipartEncoder
 import json
 
-class CHCSVWriter(Writer):
-    """Write into ClickHouse via CSV file and clickhouse-client tool"""
+class TBCSVWriter(Writer):
+    """Write into Tinybird via CSV file"""
 
     dst_schema = None
     dst_table = None
     dst_distribute = None
 
-    host = None
-    port = None
-    user = None
-    password = None
+    tb_host = None
+    tb_token = None
 
     def __init__(
             self,
-            connection_settings,
+            tb_host,
+            tb_token,
             dst_schema=None,
             dst_table=None,
             dst_table_prefix=None,
             dst_distribute=False,
     ):
-        if dst_distribute and dst_schema is not None:
-            dst_schema += "_all"
-        if dst_distribute and dst_table is not None:
-            dst_table += "_all"
-        logging.info(
-            "CHCSWriter() connection_settings={} dst_schema={} dst_table={}".format(connection_settings, dst_schema,
-                                                                                    dst_table))
-        self.host = connection_settings['host']
-        self.port = connection_settings['port']
-        self.user = connection_settings['user']
-        self.password = connection_settings['password']
+        # if dst_distribute and dst_schema is not None:
+        #     dst_schema += "_all"
+        # if dst_distribute and dst_table is not None:
+        #     dst_table += "_all"
+        # logging.info(
+        #     "CHCSWriter() connection_settings={} dst_schema={} dst_table={}".format(connection_settings, dst_schema,
+        #                                                                             dst_table))
+        self.tb_host = tb_host
+        self.tb_token = tb_token
+
+        if self.tb_host is None or self.tb_token is None:
+            logging.critical(f" Host: {self.tb_host} or token {self.tb_token} is missing")
+            return None
+
         self.dst_schema = dst_schema
         self.dst_table = dst_table
         self.dst_table_prefix = dst_table_prefix
@@ -75,17 +77,14 @@ def insert(self, event_or_events=None):
                 'name': self.dst_table,
                 'mode': 'append'
             }
-            token = 'p.eyJ1IjogIjUzMWUwMDUxLTRmZGQtNDc5MC05OGY0LWNlNTA3NDVkYzFkNSIsICJpZCI6ICIzNzMyOWExNy1mMGM2LTQyNWMtODhmOC1hZDE2MTU4OWIwNmMifQ.80_ZbRL9D8x2mlWay1EjdeA-zi7asxqKqp06NoOmadg'
-            # token = 'p.eyJ1IjogImJjYzkyYjJkLTY3YWItNDIxMC05YjkzLWVhYjc4ODA3ZTVkMiIsICJpZCI6ICJiMjNiNGQwNC01N2FhLTRiOTMtOTNkMS1lNzE0ODcxZWJkYTUifQ.YNzxkpebfTIgZGSQ9UiUpunKymeY6n7cWUY-3jgSp8E'
 
             f = open(event.filename, 'rb')
             m = MultipartEncoder(fields={'csv': ('csv', f, 'text/csv')})
 
-            url = 'https://tb-test.tinybird.co/v0/datasources'
-            # url = 'http://localhost:8001/v0/datasources'
+            url = f"{self.tb_host}/v0/datasources"
 
             response = requests.post(url, data=m,
-                            headers={'Authorization': 'Bearer ' + token, 'Content-Type': m.content_type},
+                            headers={'Authorization': 'Bearer ' + self.tb_token, 'Content-Type': m.content_type},
                             params=params
                         )