simplify flow

Author: asuncepeda

clickhouse_mysql/writer/tbcsvwriter.py

@@ -57,16 +57,14 @@ def __init__(
 
 
     def uploadCSV(self, table, filename, tries=1):
+        limit_of_retries = 9
         self.not_uploaded = False
         params = {
             'name': table,
             'mode': 'append',
             'dialect_delimiter': ','
         }
 
-        limit_of_retries = 10
-        retry_after = 0
-
         try:
             with open(filename, 'rb') as f:
                 m = MultipartEncoder(fields={'csv': ('csv', f, 'text/csv')})
@@ -92,15 +90,20 @@ def uploadCSV(self, table, filename, tries=1):
                     # retry with the recommended value
                     retry_after = int(response.headers['Retry-After']) + tries
                     logging.error(f"Too many requests retrying in {retry_after} seconds to upload {filename} to {table}")
+                    time.sleep(retry_after)
+                    self.uploadCSV(table, filename, tries + 1)
                 elif response.status_code >= 500:
                     # In case of server error let's retry `limit_of_retries` times, but exponentially
                     logging.error(response.content)
                     if tries > limit_of_retries:
+                        logging.debug(f'Limit of retries reached for {filename}')
                         self.not_uploaded = True
                         return
 
                     retry_after = 2 ** tries
                     logging.info(f"Retrying {filename} when status {response.status_code}, waiting {retry_after}, try {tries} of {limit_of_retries}")
+                    time.sleep(retry_after)
+                    self.uploadCSV(table, filename, tries + 1)
                 else:
                     # In case of other client errors (400, 404, etc...) don't retry
                     logging.error(response.content)
@@ -112,15 +115,16 @@ def uploadCSV(self, table, filename, tries=1):
             # As we don't know what went wrong... let's retry `limit_of_retries` times, but exponentially
             logging.exception(e)
             if tries > limit_of_retries:
+                logging.debug(f'Limit of retries reached for {filename}')
                 self.not_uploaded = True
                 return
 
             retry_after = 2 ** tries
             logging.info(f"Retrying {filename} when exception, waiting {retry_after} try {tries} of {limit_of_retries}")
-        finally:
             time.sleep(retry_after)
             self.uploadCSV(table, filename, tries + 1)
 
+
     def insert(self, event_or_events=None):
         # event_or_events = [
         #   event: {