feat: exponential random retry

CHANGELOG.md

@@ -3,6 +3,7 @@
 ### Features
 1. [#194](https://github.com/influxdata/influxdb-client-csharp/pull/194): Add possibility to handle HTTP response from InfluxDB server [write]
 1. [#197](https://github.com/influxdata/influxdb-client-csharp/pull/197): Optimize Flux Query for querying one time-series [LINQ]
+1. [#205](https://github.com/influxdata/influxdb-client-csharp/pull/205): Exponential random retry [write]
 
 ### Bug Fixes
 1. [#193](https://github.com/influxdata/influxdb-client-csharp/pull/193): Create services without API implementation

Client.Test/RetryAttemptTest.cs

@@ -78,7 +78,8 @@ public void HeaderHasPriority()
             Assert.AreEqual(10_000, retry.GetRetryInterval());
 
             retry = new RetryAttempt(new HttpException("", 429), 1, _default);
-            Assert.AreEqual(5_000, retry.GetRetryInterval());
+            Assert.GreaterOrEqual( retry.GetRetryInterval(), 5_000);
+            Assert.LessOrEqual( retry.GetRetryInterval(), 10_000);
         }
 
         [Test]
@@ -87,29 +88,49 @@ public void ExponentialBase()
             var options = WriteOptions.CreateNew()
                 .RetryInterval(5_000)
                 .ExponentialBase(5)
+                .MaxRetries(4)
                 .MaxRetryDelay(int.MaxValue)
                 .Build();
 
             var retry = new RetryAttempt(new HttpException("", 429), 1, options);
-            Assert.AreEqual(5_000, retry.GetRetryInterval());
+            var retryInterval = retry.GetRetryInterval();
+            Assert.GreaterOrEqual(retryInterval, 5_000);
+            Assert.LessOrEqual(retryInterval, 25_000);
+            Assert.IsTrue(retry.IsRetry());
 
             retry = new RetryAttempt(new HttpException("", 429), 2, options);
-            Assert.AreEqual(25_000, retry.GetRetryInterval());
+            retryInterval = retry.GetRetryInterval();
+            Assert.GreaterOrEqual(retryInterval, 25_000);
+            Assert.LessOrEqual(retryInterval, 125_000);
+            Assert.IsTrue(retry.IsRetry());
 
             retry = new RetryAttempt(new HttpException("", 429), 3, options);
-            Assert.AreEqual(125_000, retry.GetRetryInterval());
+            retryInterval = retry.GetRetryInterval();           
+            Assert.GreaterOrEqual(retryInterval, 125_000);
+            Assert.LessOrEqual(retryInterval, 625_000);
+            Assert.IsTrue(retry.IsRetry());
 
             retry = new RetryAttempt(new HttpException("", 429), 4, options);
-            Assert.AreEqual(625_000, retry.GetRetryInterval());
+            retryInterval = retry.GetRetryInterval();
+            Assert.GreaterOrEqual(retryInterval, 625_000);
+            Assert.LessOrEqual(retryInterval, 3_125_000);
+            Assert.IsTrue(retry.IsRetry());
 
             retry = new RetryAttempt(new HttpException("", 429), 5, options);
-            Assert.AreEqual(3_125_000, retry.GetRetryInterval());
+            retryInterval = retry.GetRetryInterval();
+            Assert.GreaterOrEqual(retryInterval, 3_125_000);
+            Assert.LessOrEqual(retryInterval, 15_625_000);
+            Assert.IsFalse(retry.IsRetry());
 
             retry = new RetryAttempt(new HttpException("", 429), 6, options);
-            Assert.AreEqual(15_625_000, retry.GetRetryInterval());
+            retryInterval = retry.GetRetryInterval();
+            Assert.GreaterOrEqual(retryInterval, 15_625_000);
+            Assert.LessOrEqual(retryInterval, 78_125_000);
+            Assert.IsFalse(retry.IsRetry());
 
             retry = new RetryAttempt(CreateException(3), 7, options);
-            Assert.AreEqual(3_000, retry.GetRetryInterval());
+            retryInterval = retry.GetRetryInterval();
+            Assert.AreEqual(3_000, retryInterval);
         }
 
         [Test]
@@ -118,26 +139,32 @@ public void MaxRetryDelay()
             var options = WriteOptions.CreateNew()
                 .RetryInterval(2_000)
                 .ExponentialBase(2)
+                .MaxRetries(10)
                 .MaxRetryDelay(50_000)
                 .Build();
 
             var retry = new RetryAttempt(new HttpException("", 429), 1, options);
-            Assert.AreEqual(2_000, retry.GetRetryInterval());
+            Assert.GreaterOrEqual(retry.GetRetryInterval(), 2_000);
+            Assert.LessOrEqual(retry.GetRetryInterval(), 4_000);
 
             retry = new RetryAttempt(new HttpException("", 429), 2, options);
-            Assert.AreEqual(4_000, retry.GetRetryInterval());
+            Assert.GreaterOrEqual(retry.GetRetryInterval(), 4_000);
+            Assert.LessOrEqual(retry.GetRetryInterval(), 8_000);
 
             retry = new RetryAttempt(new HttpException("", 429), 3, options);
-            Assert.AreEqual(8_000, retry.GetRetryInterval());
+            Assert.GreaterOrEqual(retry.GetRetryInterval(), 8_000);
+            Assert.LessOrEqual(retry.GetRetryInterval(), 16_000);
 
             retry = new RetryAttempt(new HttpException("", 429), 4, options);
-            Assert.AreEqual(16_000, retry.GetRetryInterval());
+            Assert.GreaterOrEqual(retry.GetRetryInterval(), 16_000);
+            Assert.LessOrEqual(retry.GetRetryInterval(), 32_000);
 
             retry = new RetryAttempt(new HttpException("", 429), 5, options);
-            Assert.AreEqual(32_000, retry.GetRetryInterval());
+            Assert.GreaterOrEqual(retry.GetRetryInterval(), 32_000);
+            Assert.LessOrEqual(retry.GetRetryInterval(), 50_000);
 
             retry = new RetryAttempt(new HttpException("", 429), 6, options);
-            Assert.AreEqual(50_000, retry.GetRetryInterval());
+            Assert.LessOrEqual(retry.GetRetryInterval(), 50_000);
         }
 
         private HttpException CreateException(int retryAfter = 10)

Client.Test/WriteApiTest.cs

@@ -145,7 +145,8 @@ public void Retry()
             var retriableErrorEvent = listener.Get<WriteRetriableErrorEvent>();
             Assert.AreEqual("token is temporarily over quota", retriableErrorEvent.Exception.Message);
             Assert.AreEqual(429, ((HttpException) retriableErrorEvent.Exception).Status);
-            Assert.AreEqual(1000, retriableErrorEvent.RetryInterval);
+            Assert.GreaterOrEqual(retriableErrorEvent.RetryInterval, 1000);
+            Assert.LessOrEqual(retriableErrorEvent.RetryInterval, 2000);
 
             //
             // Second request success
@@ -422,9 +423,9 @@ public void WriteOptionsDefaults()
             var options = WriteOptions.CreateNew().Build();
 
             Assert.AreEqual(5_000, options.RetryInterval);
-            Assert.AreEqual(3, options.MaxRetries);
-            Assert.AreEqual(180_000, options.MaxRetryDelay);
-            Assert.AreEqual(5, options.ExponentialBase);
+            Assert.AreEqual(5, options.MaxRetries);
+            Assert.AreEqual(125_000, options.MaxRetryDelay);
+            Assert.AreEqual(2, options.ExponentialBase);
         }
 
         [Test]

Client/Internal/RetryAttempt.cs

@@ -31,6 +31,7 @@ internal class RetryAttempt
         internal Exception Error { get; }
         private readonly int _count;
         private readonly WriteOptions _writeOptions;
+        private readonly Random _random = new Random();
 
         internal RetryAttempt(Exception error, int count, WriteOptions writeOptions)
         {
@@ -82,25 +83,37 @@ internal bool IsRetry()
         /// <returns>retry interval to sleep</returns>
         internal long GetRetryInterval()
         {
-            long retryInterval;
-
             // from header
             if (Error is HttpException httpException && httpException.RetryAfter.HasValue)
             {
-                retryInterval = httpException.RetryAfter.Value * 1000;
+                return httpException.RetryAfter.Value * 1000 + JitterDelay(_writeOptions);
             }
+
             // from configuration
-            else
+            var rangeStart = _writeOptions.RetryInterval;
+            var rangeStop = _writeOptions.RetryInterval * _writeOptions.ExponentialBase;
+
+            var i = 1;
+            while (i < _count)
             {
-                retryInterval = _writeOptions.RetryInterval
-                                * (long) (Math.Pow(_writeOptions.ExponentialBase, _count - 1));
-                retryInterval = Math.Min(retryInterval, _writeOptions.MaxRetryDelay);
+                i++;
+                rangeStart = rangeStop;
+                rangeStop = rangeStop * _writeOptions.ExponentialBase;
+                if (rangeStop > _writeOptions.MaxRetryDelay)
+                {
+                    break;
+                }
+            }
 
-                Trace.WriteLine($"The InfluxDB does not specify \"Retry-After\". " +
-                                $"Use the default retryInterval: {retryInterval}");
+            if (rangeStop > _writeOptions.MaxRetryDelay)
+            {
+                rangeStop = _writeOptions.MaxRetryDelay;
             }
 
-            retryInterval += JitterDelay(_writeOptions);
+            var retryInterval = _random.Next(rangeStart, rangeStop);
+
+            Trace.WriteLine($"The InfluxDB does not specify \"Retry-After\". " +
+                            $"Use the default retryInterval: {retryInterval}");
 
             return retryInterval;
         }

Client/README.md

@@ -368,8 +368,8 @@ The writes are processed in batches which are configurable by `WriteOptions`:
 | **JitterInterval** | the number of milliseconds to increase the batch flush interval by a random amount| 0 |
 | **RetryInterval** | the number of milliseconds to retry unsuccessful write. The retry interval is used when the InfluxDB server does not specify "Retry-After" header. | 5000 |
 | **MaxRetries** | the number of max retries when write fails | 3 |
-| **MaxRetryDelay** | the maximum delay between each retry attempt in milliseconds | 180_000 |
-| **ExponentialBase** |  the base for the exponential retry delay, the next delay is computed as `RetryInterval * ExponentialBase^(attempts-1) + random(JitterInterval)` | 5 |
+| **MaxRetryDelay** | the maximum delay between each retry attempt in milliseconds | 125_000 |
+| **ExponentialBase** |  the base for the exponential retry delay, the next delay is computed using random exponential backoff as a random value within the interval  ``retryInterval * exponentialBase^(attempts-1)`` and ``retryInterval * exponentialBase^(attempts)``. Example for ``retryInterval=5_000, exponentialBase=2, maxRetryDelay=125_000, maxRetries=5`` Retry delays are random distributed values within the ranges of ``[5_000-10_000, 10_000-20_000, 20_000-40_000, 40_000-80_000, 80_000-125_000]`` | 2 |
 
 ### Writing data
 

Client/WriteOptions.cs

@@ -23,9 +23,9 @@ public class WriteOptions
         private const int DefaultFlushInterval = 1000;
         private const int DefaultJitterInterval = 0;
         private const int DefaultRetryInterval = 5000;
-        private const int DefaultMaxRetries = 3;
-        private const int DefaultMaxRetryDelay = 180_000;
-        private const int DefaultExponentialBase = 5;
+        private const int DefaultMaxRetries = 5;
+        private const int DefaultMaxRetryDelay = 125_000;
+        private const int DefaultExponentialBase = 2;
 
         /// <summary>
         /// The number of data point to collect in batch.