feat: add exponential backoff strategy for retry (influxdata#140)

Author: bednar

CHANGELOG.md

@@ -1,6 +1,7 @@
 ## 1.10.0 [unreleased]
 
 ### Features
+1. [#140](https://github.com/influxdata/influxdb-client-python/pull/140): Added exponential backoff strategy for batching writes, Allowed to configure default retry strategy. Default value for `retry_interval` is 5_000 milliseconds.   
 1. [#136](https://github.com/influxdata/influxdb-client-python/pull/136): Allows users to skip of verifying SSL certificate
 1. [#143](https://github.com/influxdata/influxdb-client-python/pull/143): Skip of verifying SSL certificate could be configured via config file or environment properties
 1. [#141](https://github.com/influxdata/influxdb-client-python/pull/141): Added possibility to use datetime nanoseconds precision by `pandas.Timestamp`  

README.rst

@@ -250,7 +250,16 @@ The batching is configurable by ``write_options``\ :
      - ``0``
    * - **retry_interval**
      - the number of milliseconds to retry unsuccessful write. The retry interval is used when the InfluxDB server does not specify "Retry-After" header.
-     - ``1000``
+     - ``5000``
+   * - **max_retries**
+     - the number of max retries when write fails
+     - ``3``
+   * - **max_retry_delay**
+     - the maximum delay between each retry attempt in milliseconds
+     - ``180_000``
+   * - **exponential_base**
+     - the base for the exponential retry delay, the next delay is computed as ``retry_interval * exponential_base^(attempts-1) + random(jitter_interval)``
+     - ``5``
 
 
 .. code-block:: python
@@ -265,7 +274,10 @@ The batching is configurable by ``write_options``\ :
    _write_client = _client.write_api(write_options=WriteOptions(batch_size=500,
                                                                 flush_interval=10_000,
                                                                 jitter_interval=2_000,
-                                                                retry_interval=5_000))
+                                                                retry_interval=5_000,
+                                                                max_retries=5,
+                                                                max_retry_delay=30_000,
+                                                                exponential_base=2))
 
    """
    Write Line Protocol formatted as string
@@ -899,6 +911,23 @@ The following forward compatible APIs are available:
 
 For detail info see `InfluxDB 1.8 example <examples/influxdb_18_example.py>`_.
 
+HTTP Retry Strategy
+^^^^^^^^^^^^^^^^^^^
+By default the client uses a retry strategy only for batching writes (for more info see `Batching`_).
+For other HTTP requests there is no one retry strategy, but it could be configured by ``retries``
+parameter of ``InfluxDBClient``.
+
+For more info about how configure HTTP retry see details in `urllib3 documentation <https://urllib3.readthedocs.io/en/latest/reference/index.html?highlight=retry#urllib3.Retry>`_.
+
+.. code-block:: python
+
+    from urllib3 import Retry
+
+    from influxdb_client import InfluxDBClient
+
+    retries = Retry(connect=5, read=2, redirect=5)
+    client = InfluxDBClient(url="http://localhost:9999", token="my-token", org="my-org", retries=retries)
+
 Nanosecond precision
 ^^^^^^^^^^^^^^^^^^^^
 

influxdb_client/api_client.py

@@ -63,14 +63,14 @@ class ApiClient(object):
     _pool = None
 
     def __init__(self, configuration=None, header_name=None, header_value=None,
-                 cookie=None, pool_threads=None):
+                 cookie=None, pool_threads=None, retries=False):
         """Initialize generic API client."""
         if configuration is None:
             configuration = Configuration()
         self.configuration = configuration
         self.pool_threads = pool_threads
 
-        self.rest_client = rest.RESTClientObject(configuration)
+        self.rest_client = rest.RESTClientObject(configuration, retries=retries)
         self.default_headers = {}
         if header_name is not None:
             self.default_headers[header_name] = header_value
@@ -113,7 +113,7 @@ def __call_api(
             query_params=None, header_params=None, body=None, post_params=None,
             files=None, response_type=None, auth_settings=None,
             _return_http_data_only=None, collection_formats=None,
-            _preload_content=True, _request_timeout=None):
+            _preload_content=True, _request_timeout=None, urlopen_kw=None):
 
         config = self.configuration
 
@@ -164,12 +164,14 @@ def __call_api(
         # request url
         url = self.configuration.host + resource_path
 
+        urlopen_kw = urlopen_kw or {}
+
         # perform request and return response
         response_data = self.request(
             method, url, query_params=query_params, headers=header_params,
             post_params=post_params, body=body,
             _preload_content=_preload_content,
-            _request_timeout=_request_timeout)
+            _request_timeout=_request_timeout, **urlopen_kw)
 
         self.last_response = response_data
 
@@ -295,7 +297,7 @@ def call_api(self, resource_path, method,
                  body=None, post_params=None, files=None,
                  response_type=None, auth_settings=None, async_req=None,
                  _return_http_data_only=None, collection_formats=None,
-                 _preload_content=True, _request_timeout=None):
+                 _preload_content=True, _request_timeout=None, urlopen_kw=None):
         """Make the HTTP request (synchronous) and Return deserialized data.
 
         To make an async_req request, set the async_req parameter.
@@ -325,6 +327,8 @@ def call_api(self, resource_path, method,
                                  number provided, it will be total request
                                  timeout. It can also be a pair (tuple) of
                                  (connection, read) timeouts.
+        :param urlopen_kw: Additional parameters are passed to
+                           :meth:`urllib3.request.RequestMethods.request`
         :return:
             If async_req parameter is True,
             the request will be called asynchronously.
@@ -338,7 +342,7 @@ def call_api(self, resource_path, method,
                                    body, post_params, files,
                                    response_type, auth_settings,
                                    _return_http_data_only, collection_formats,
-                                   _preload_content, _request_timeout)
+                                   _preload_content, _request_timeout, urlopen_kw)
         else:
             thread = self.pool.apply_async(self.__call_api, (resource_path,
                                            method, path_params, query_params,
@@ -347,64 +351,71 @@ def call_api(self, resource_path, method,
                                            response_type, auth_settings,
                                            _return_http_data_only,
                                            collection_formats,
-                                           _preload_content, _request_timeout))
+                                           _preload_content, _request_timeout, urlopen_kw))
         return thread
 
     def request(self, method, url, query_params=None, headers=None,
                 post_params=None, body=None, _preload_content=True,
-                _request_timeout=None):
+                _request_timeout=None, **urlopen_kw):
         """Make the HTTP request using RESTClient."""
         if method == "GET":
             return self.rest_client.GET(url,
                                         query_params=query_params,
                                         _preload_content=_preload_content,
                                         _request_timeout=_request_timeout,
-                                        headers=headers)
+                                        headers=headers,
+                                        **urlopen_kw)
         elif method == "HEAD":
             return self.rest_client.HEAD(url,
                                          query_params=query_params,
                                          _preload_content=_preload_content,
                                          _request_timeout=_request_timeout,
-                                         headers=headers)
+                                         headers=headers,
+                                         **urlopen_kw)
         elif method == "OPTIONS":
             return self.rest_client.OPTIONS(url,
                                             query_params=query_params,
                                             headers=headers,
                                             post_params=post_params,
                                             _preload_content=_preload_content,
                                             _request_timeout=_request_timeout,
-                                            body=body)
+                                            body=body,
+                                            **urlopen_kw)
         elif method == "POST":
             return self.rest_client.POST(url,
                                          query_params=query_params,
                                          headers=headers,
                                          post_params=post_params,
                                          _preload_content=_preload_content,
                                          _request_timeout=_request_timeout,
-                                         body=body)
+                                         body=body,
+                                         **urlopen_kw)
         elif method == "PUT":
             return self.rest_client.PUT(url,
                                         query_params=query_params,
                                         headers=headers,
                                         post_params=post_params,
                                         _preload_content=_preload_content,
                                         _request_timeout=_request_timeout,
-                                        body=body)
+                                        body=body,
+                                        **urlopen_kw)
         elif method == "PATCH":
             return self.rest_client.PATCH(url,
                                           query_params=query_params,
                                           headers=headers,
                                           post_params=post_params,
                                           _preload_content=_preload_content,
                                           _request_timeout=_request_timeout,
-                                          body=body)
+                                          body=body,
+                                          **urlopen_kw)
         elif method == "DELETE":
             return self.rest_client.DELETE(url,
                                            query_params=query_params,
                                            headers=headers,
                                            _preload_content=_preload_content,
                                            _request_timeout=_request_timeout,
-                                           body=body)
+                                           body=body,
+                                           **urlopen_kw)
         else:
             raise ValueError(
                 "http method must be `GET`, `HEAD`, `OPTIONS`,"

influxdb_client/client/influxdb_client.py

@@ -32,7 +32,9 @@ def __init__(self, url, token, debug=None, timeout=10000, enable_gzip=False, org
         :param enable_gzip: Enable Gzip compression for http requests. Currently only the "Write" and "Query" endpoints
                             supports the Gzip compression.
         :param org: organization name (used as a default in query and write API)
-        :key verify_ssl: Set this to false to skip verifying SSL certificate when calling API from https server.
+        :key bool verify_ssl: Set this to false to skip verifying SSL certificate when calling API from https server.
+        :key urllib3.util.retry.Retry retries: Set the default retry strategy that is used for all HTTP requests
+                                               except batching writes. As a default there is no one retry strategy.
 
         """
         self.url = url
@@ -55,8 +57,10 @@ def __init__(self, url, token, debug=None, timeout=10000, enable_gzip=False, org
         auth_header_name = "Authorization"
         auth_header_value = "Token " + auth_token
 
+        retries = kwargs.get('retries', False)
+
         self.api_client = ApiClient(configuration=conf, header_name=auth_header_name,
-                                    header_value=auth_header_value)
+                                    header_value=auth_header_value, retries=retries)
 
     @classmethod
     def from_config_file(cls, config_file: str = "config.ini", debug=None, enable_gzip=False):
@@ -217,7 +221,6 @@ def health(self) -> HealthCheck:
             health = health_service.get_health()
             return health
         except Exception as e:
-            print(e)
             return HealthCheck(name="influxdb", message=str(e), status="fail")
 
     def ready(self) -> Ready:

influxdb_client/client/write/retry.py

@@ -0,0 +1,67 @@
+"""Implementation for Retry strategy during HTTP requests."""
+
+from itertools import takewhile
+from random import random
+
+from urllib3 import Retry
+
+
+class WritesRetry(Retry):
+    """
+    Writes retry configuration.
+
+    :param int jitter_interval: random milliseconds when retrying writes
+    :param int max_retry_delay: maximum delay when retrying write
+    :param int exponential_base: base for the exponential retry delay, the next delay is computed as
+                                 `backoff_factor * exponential_base^(attempts-1) + random(jitter_interval)`
+    """
+
+    def __init__(self, jitter_interval=0, max_retry_delay=180, exponential_base=5, **kw):
+        """Initialize defaults."""
+        super().__init__(**kw)
+        self.jitter_interval = jitter_interval
+        self.max_retry_delay = max_retry_delay
+        self.exponential_base = exponential_base
+
+    def new(self, **kw):
+        """Initialize defaults."""
+        if 'jitter_interval' not in kw:
+            kw['jitter_interval'] = self.jitter_interval
+        if 'max_retry_delay' not in kw:
+            kw['max_retry_delay'] = self.max_retry_delay
+        if 'exponential_base' not in kw:
+            kw['exponential_base'] = self.exponential_base
+        return super().new(**kw)
+
+    def is_retry(self, method, status_code, has_retry_after=False):
+        """is_retry doesn't require retry_after header. If there is not Retry-After we will use backoff."""
+        if not self._is_method_retryable(method):
+            return False
+
+        return self.total and (status_code >= 429)
+
+    def get_backoff_time(self):
+        """Variant of exponential backoff with initial and max delay and a random jitter delay."""
+        # We want to consider only the last consecutive errors sequence (Ignore redirects).
+        consecutive_errors_len = len(
+            list(
+                takewhile(lambda x: x.redirect_location is None, reversed(self.history))
+            )
+        )
+        # First fail doesn't increase backoff
+        consecutive_errors_len -= 1
+        if consecutive_errors_len < 0:
+            return 0
+
+        backoff_value = self.backoff_factor * (self.exponential_base ** consecutive_errors_len) + self._jitter_delay()
+        return min(self.max_retry_delay, backoff_value)
+
+    def get_retry_after(self, response):
+        """Get the value of Retry-After header and append random jitter delay."""
+        retry_after = super().get_retry_after(response)
+        if retry_after:
+            retry_after += self._jitter_delay()
+        return retry_after
+
+    def _jitter_delay(self):
+        return self.jitter_interval * random()