Changed update to include in alter table command just those columns which actually change

Author: ygnuss

clickhouse_mysql/writer/chwriter.py

@@ -283,19 +283,33 @@ def update(self, event_or_events):
                                                                                        self.dst_table))
 
         # and UPDATE converted rows
-
+        # improve performance updating just those fields which have actually changed
+        updated_values = dict(set(row['after_values'].items()).difference(set(row['before_values'].items())))
+        
         sql = ''
         try:
+            # sql = 'ALTER TABLE `{0}`.`{1}` UPDATE {2}, `tb_upd`={3} where {4}'.format(
+            #     schema,
+            #     table,
+            #     ', '.join(filter(None, map(lambda column, value: "" if column in pk or value is None else self.get_data_format(column, value), row['after_values'].keys(), row['after_values'].values()))),
+            #     "'%s'" % datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
+            #     ' and '.join(filter(None, map(
+            #         lambda column, value: "" if column not in pk or value is None else self.get_data_format(column, value),
+            #         row['before_values'].keys(), row['before_values'].values())))
+            # )
+
             sql = 'ALTER TABLE `{0}`.`{1}` UPDATE {2}, `tb_upd`={3} where {4}'.format(
                 schema,
                 table,
-                ', '.join(filter(None, map(lambda column, value: "" if column in pk or value is None else self.get_data_format(column, value), row['after_values'].keys(), row['after_values'].values()))),
+                ', '.join(filter(None, map(lambda column, value: "" if column in pk or value is None else self.get_data_format(column, value), updated_values.keys(), updated_values.values()))),
                 "'%s'" % datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
                 ' and '.join(filter(None, map(
                     lambda column, value: "" if column not in pk or value is None else self.get_data_format(column, value),
                     row['before_values'].keys(), row['before_values'].values())))
             )
 
+            logging.debug("SQL UPDATE: \n\n " + sql + "\n\n")
+
             self.client.execute(sql)
         except Exception as ex:
             logging.critical('QUERY FAILED')

requirements.txt

@@ -1,108 +1,6 @@
-appdirs==1.4.4
-apturl==0.5.2
-astroid==2.4.2
-basictracer==3.1.0
-blinker==1.4
-Brlapi==0.7.0
-cachetools==4.0.0
-certifi==2020.12.5
-chardet==3.0.4
-chrome-gnome-shell==0.0.0
-Click==7.0
-clickhouse-driver==0.0.10
-clickhouse-toolset @ file:///tmp/clickhouse_toolset-0.9.dev0-cp38-cp38-linux_x86_64.whl
-colorama==0.4.3
-command-not-found==0.3
-configobj==5.0.6
-crcmod==1.7
-cryptography==3.0
-cupshelpers==1.0
-datasketch==1.2.10
-dbus-python==1.2.16
-defer==1.0.6
-distlib==0.3.1
-distro==1.5.0
-distro-info===0.23ubuntu1
-dlib==19.16.0
-filelock==3.0.12
-httplib2==0.18.1
-humanfriendly==8.2
-idna==2.6
-importlib-metadata==1.6.0
-isort==5.7.0
-jeepney==0.4.3
-keyring==21.3.0
-language-selector==0.1
-launchpadlib==1.10.13
-lazr.restfulclient==0.14.2
-lazr.uri==1.0.5
-lazy-object-proxy==1.4.3
-louis==3.14.0
-macaroonbakery==1.3.1
-Markdown==3.2.1
-mccabe==0.6.1
-more-itertools==4.2.0
-msal==1.5.0
-netifaces==0.10.4
-numpy==1.15.0
-oauthlib==3.1.0
-olefile==0.46
-opencv-python==4.4.0.46
-opentracing==2.0.0
-passlib==1.7.1
-pexpect==4.6.0
-Pillow==7.2.0
-powerline-status==2.8.1
-protobuf==3.12.3
-psutil==5.6.3
-psycopg2-binary==2.8.5
-pycairo==1.16.2
-pycups==2.0.1
-pycurl==7.43.0.6
-Pygments==2.3.1
-PyGObject==3.38.0
-PyJWT==1.6.4
-pylint==2.6.0
-pymacaroons==0.13.0
+clickhouse-driver==0.2.0
+mysql-replication==0.23
+mysqlclient==2.0.3
 PyMySQL==1.0.2
-PyNaCl==1.4.0
-pyRFC3339==1.1
-python-apt==2.1.3+ubuntu1.3
-python-dateutil==2.8.1
-python-debian==0.1.37
-pytz==2020.1
-pyxdg==0.26
-PyYAML==5.3.1
-rangehttpserver==1.2.0
-redis==3.2.1
-reportlab==3.5.47
-requests==2.18.4
-requests-toolbelt==0.9.1
-requests-unixsocket==0.2.0
-screen-resolution-extra==0.0.0
-SecretStorage==3.1.2
-simplejson==3.17.0
-six==1.15.0
-streaming-form-data==1.1.0
-systemd-python==234
-tabulate==0.8.3
-terminator==1.92
--e [email protected]:tinybird/analytics.git@0d13783b7e38c0decc97ac06901e8ce7b804221e#egg=tinybird
-tinybird-cli==1.0.0b12
-TLPUI==1.3.1.post3
-toml==0.10.2
-toposort==1.5
-tornado==5.1.1
-tornado-opentracing==1.0.1
-torngithub==0.2.0
-ubuntu-advantage-tools==24.4
-ubuntu-drivers-common==0.0.0
-ufw==0.36
-unattended-upgrades==0.1
-urllib3==1.22
-vboxapi==1.0
-virtualenv==20.0.29+ds
-wadllib==1.3.4
-wrapt==1.12.1
-xkit==0.0.0
-zipp==1.0.0
+pytz==2021.1
+tzlocal==2.1