Skip to content

Commit 73eb2c5

Browse files
rhajekrhajek
authored
feat: exponential random retry (#205)
* feat: exponential random retry
1 parent bbf9f29 commit 73eb2c5

File tree

6 files changed

+75
-33
lines changed

6 files changed

+75
-33
lines changed

‎CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
### Features
44
1. [#194](https://github.com/influxdata/influxdb-client-csharp/pull/194): Add possibility to handle HTTP response from InfluxDB server [write]
55
1. [#197](https://github.com/influxdata/influxdb-client-csharp/pull/197): Optimize Flux Query for querying one time-series [LINQ]
6+
1. [#205](https://github.com/influxdata/influxdb-client-csharp/pull/205): Exponential random retry [write]
67

78
### Bug Fixes
89
1. [#193](https://github.com/influxdata/influxdb-client-csharp/pull/193): Create services without API implementation

‎Client.Test/RetryAttemptTest.cs

Lines changed: 41 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,8 @@ public void HeaderHasPriority()
7878
Assert.AreEqual(10_000, retry.GetRetryInterval());
7979

8080
retry = new RetryAttempt(new HttpException("", 429), 1, _default);
81-
Assert.AreEqual(5_000, retry.GetRetryInterval());
81+
Assert.GreaterOrEqual( retry.GetRetryInterval(), 5_000);
82+
Assert.LessOrEqual( retry.GetRetryInterval(), 10_000);
8283
}
8384

8485
[Test]
@@ -87,29 +88,49 @@ public void ExponentialBase()
8788
var options = WriteOptions.CreateNew()
8889
.RetryInterval(5_000)
8990
.ExponentialBase(5)
91+
.MaxRetries(4)
9092
.MaxRetryDelay(int.MaxValue)
9193
.Build();
9294

9395
var retry = new RetryAttempt(new HttpException("", 429), 1, options);
94-
Assert.AreEqual(5_000, retry.GetRetryInterval());
96+
var retryInterval = retry.GetRetryInterval();
97+
Assert.GreaterOrEqual(retryInterval, 5_000);
98+
Assert.LessOrEqual(retryInterval, 25_000);
99+
Assert.IsTrue(retry.IsRetry());
95100

96101
retry = new RetryAttempt(new HttpException("", 429), 2, options);
97-
Assert.AreEqual(25_000, retry.GetRetryInterval());
102+
retryInterval = retry.GetRetryInterval();
103+
Assert.GreaterOrEqual(retryInterval, 25_000);
104+
Assert.LessOrEqual(retryInterval, 125_000);
105+
Assert.IsTrue(retry.IsRetry());
98106

99107
retry = new RetryAttempt(new HttpException("", 429), 3, options);
100-
Assert.AreEqual(125_000, retry.GetRetryInterval());
108+
retryInterval = retry.GetRetryInterval();
109+
Assert.GreaterOrEqual(retryInterval, 125_000);
110+
Assert.LessOrEqual(retryInterval, 625_000);
111+
Assert.IsTrue(retry.IsRetry());
101112

102113
retry = new RetryAttempt(new HttpException("", 429), 4, options);
103-
Assert.AreEqual(625_000, retry.GetRetryInterval());
114+
retryInterval = retry.GetRetryInterval();
115+
Assert.GreaterOrEqual(retryInterval, 625_000);
116+
Assert.LessOrEqual(retryInterval, 3_125_000);
117+
Assert.IsTrue(retry.IsRetry());
104118

105119
retry = new RetryAttempt(new HttpException("", 429), 5, options);
106-
Assert.AreEqual(3_125_000, retry.GetRetryInterval());
120+
retryInterval = retry.GetRetryInterval();
121+
Assert.GreaterOrEqual(retryInterval, 3_125_000);
122+
Assert.LessOrEqual(retryInterval, 15_625_000);
123+
Assert.IsFalse(retry.IsRetry());
107124

108125
retry = new RetryAttempt(new HttpException("", 429), 6, options);
109-
Assert.AreEqual(15_625_000, retry.GetRetryInterval());
126+
retryInterval = retry.GetRetryInterval();
127+
Assert.GreaterOrEqual(retryInterval, 15_625_000);
128+
Assert.LessOrEqual(retryInterval, 78_125_000);
129+
Assert.IsFalse(retry.IsRetry());
110130

111131
retry = new RetryAttempt(CreateException(3), 7, options);
112-
Assert.AreEqual(3_000, retry.GetRetryInterval());
132+
retryInterval = retry.GetRetryInterval();
133+
Assert.AreEqual(3_000, retryInterval);
113134
}
114135

115136
[Test]
@@ -118,26 +139,32 @@ public void MaxRetryDelay()
118139
var options = WriteOptions.CreateNew()
119140
.RetryInterval(2_000)
120141
.ExponentialBase(2)
142+
.MaxRetries(10)
121143
.MaxRetryDelay(50_000)
122144
.Build();
123145

124146
var retry = new RetryAttempt(new HttpException("", 429), 1, options);
125-
Assert.AreEqual(2_000, retry.GetRetryInterval());
147+
Assert.GreaterOrEqual(retry.GetRetryInterval(), 2_000);
148+
Assert.LessOrEqual(retry.GetRetryInterval(), 4_000);
126149

127150
retry = new RetryAttempt(new HttpException("", 429), 2, options);
128-
Assert.AreEqual(4_000, retry.GetRetryInterval());
151+
Assert.GreaterOrEqual(retry.GetRetryInterval(), 4_000);
152+
Assert.LessOrEqual(retry.GetRetryInterval(), 8_000);
129153

130154
retry = new RetryAttempt(new HttpException("", 429), 3, options);
131-
Assert.AreEqual(8_000, retry.GetRetryInterval());
155+
Assert.GreaterOrEqual(retry.GetRetryInterval(), 8_000);
156+
Assert.LessOrEqual(retry.GetRetryInterval(), 16_000);
132157

133158
retry = new RetryAttempt(new HttpException("", 429), 4, options);
134-
Assert.AreEqual(16_000, retry.GetRetryInterval());
159+
Assert.GreaterOrEqual(retry.GetRetryInterval(), 16_000);
160+
Assert.LessOrEqual(retry.GetRetryInterval(), 32_000);
135161

136162
retry = new RetryAttempt(new HttpException("", 429), 5, options);
137-
Assert.AreEqual(32_000, retry.GetRetryInterval());
163+
Assert.GreaterOrEqual(retry.GetRetryInterval(), 32_000);
164+
Assert.LessOrEqual(retry.GetRetryInterval(), 50_000);
138165

139166
retry = new RetryAttempt(new HttpException("", 429), 6, options);
140-
Assert.AreEqual(50_000, retry.GetRetryInterval());
167+
Assert.LessOrEqual(retry.GetRetryInterval(), 50_000);
141168
}
142169

143170
private HttpException CreateException(int retryAfter = 10)

‎Client.Test/WriteApiTest.cs

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -145,7 +145,8 @@ public void Retry()
145145
var retriableErrorEvent = listener.Get<WriteRetriableErrorEvent>();
146146
Assert.AreEqual("token is temporarily over quota", retriableErrorEvent.Exception.Message);
147147
Assert.AreEqual(429, ((HttpException) retriableErrorEvent.Exception).Status);
148-
Assert.AreEqual(1000, retriableErrorEvent.RetryInterval);
148+
Assert.GreaterOrEqual(retriableErrorEvent.RetryInterval, 1000);
149+
Assert.LessOrEqual(retriableErrorEvent.RetryInterval, 2000);
149150

150151
//
151152
// Second request success
@@ -422,9 +423,9 @@ public void WriteOptionsDefaults()
422423
var options = WriteOptions.CreateNew().Build();
423424

424425
Assert.AreEqual(5_000, options.RetryInterval);
425-
Assert.AreEqual(3, options.MaxRetries);
426-
Assert.AreEqual(180_000, options.MaxRetryDelay);
427-
Assert.AreEqual(5, options.ExponentialBase);
426+
Assert.AreEqual(5, options.MaxRetries);
427+
Assert.AreEqual(125_000, options.MaxRetryDelay);
428+
Assert.AreEqual(2, options.ExponentialBase);
428429
}
429430

430431
[Test]

‎Client/Internal/RetryAttempt.cs

Lines changed: 23 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ internal class RetryAttempt
3131
internal Exception Error { get; }
3232
private readonly int _count;
3333
private readonly WriteOptions _writeOptions;
34+
private readonly Random _random = new Random();
3435

3536
internal RetryAttempt(Exception error, int count, WriteOptions writeOptions)
3637
{
@@ -82,25 +83,37 @@ internal bool IsRetry()
8283
/// <returns>retry interval to sleep</returns>
8384
internal long GetRetryInterval()
8485
{
85-
long retryInterval;
86-
8786
// from header
8887
if (Error is HttpException httpException && httpException.RetryAfter.HasValue)
8988
{
90-
retryInterval = httpException.RetryAfter.Value * 1000;
89+
return httpException.RetryAfter.Value * 1000 + JitterDelay(_writeOptions);
9190
}
91+
9292
// from configuration
93-
else
93+
var rangeStart = _writeOptions.RetryInterval;
94+
var rangeStop = _writeOptions.RetryInterval * _writeOptions.ExponentialBase;
95+
96+
var i = 1;
97+
while (i < _count)
9498
{
95-
retryInterval = _writeOptions.RetryInterval
96-
* (long) (Math.Pow(_writeOptions.ExponentialBase, _count - 1));
97-
retryInterval = Math.Min(retryInterval, _writeOptions.MaxRetryDelay);
99+
i++;
100+
rangeStart = rangeStop;
101+
rangeStop = rangeStop * _writeOptions.ExponentialBase;
102+
if (rangeStop > _writeOptions.MaxRetryDelay)
103+
{
104+
break;
105+
}
106+
}
98107

99-
Trace.WriteLine($"The InfluxDB does not specify \"Retry-After\". " +
100-
$"Use the default retryInterval: {retryInterval}");
108+
if (rangeStop > _writeOptions.MaxRetryDelay)
109+
{
110+
rangeStop = _writeOptions.MaxRetryDelay;
101111
}
102112

103-
retryInterval += JitterDelay(_writeOptions);
113+
var retryInterval = (long) (rangeStart + (rangeStop - rangeStart) * _random.NextDouble());
114+
115+
Trace.WriteLine("The InfluxDB does not specify \"Retry-After\". " +
116+
$"Use the default retryInterval: {retryInterval}");
104117

105118
return retryInterval;
106119
}

‎Client/README.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -368,8 +368,8 @@ The writes are processed in batches which are configurable by `WriteOptions`:
368368
| **JitterInterval** | the number of milliseconds to increase the batch flush interval by a random amount| 0 |
369369
| **RetryInterval** | the number of milliseconds to retry unsuccessful write. The retry interval is used when the InfluxDB server does not specify "Retry-After" header. | 5000 |
370370
| **MaxRetries** | the number of max retries when write fails | 3 |
371-
| **MaxRetryDelay** | the maximum delay between each retry attempt in milliseconds | 180_000 |
372-
| **ExponentialBase** | the base for the exponential retry delay, the next delay is computed as `RetryInterval * ExponentialBase^(attempts-1) + random(JitterInterval)` | 5 |
371+
| **MaxRetryDelay** | the maximum delay between each retry attempt in milliseconds | 125_000 |
372+
| **ExponentialBase** | the base for the exponential retry delay, the next delay is computed using random exponential backoff as a random value within the interval ``retryInterval * exponentialBase^(attempts-1)`` and ``retryInterval * exponentialBase^(attempts)``. Example for ``retryInterval=5_000, exponentialBase=2, maxRetryDelay=125_000, maxRetries=5`` Retry delays are random distributed values within the ranges of ``[5_000-10_000, 10_000-20_000, 20_000-40_000, 40_000-80_000, 80_000-125_000]`` | 2 |
373373

374374
### Writing data
375375

‎Client/WriteOptions.cs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,9 @@ public class WriteOptions
2323
private const int DefaultFlushInterval = 1000;
2424
private const int DefaultJitterInterval = 0;
2525
private const int DefaultRetryInterval = 5000;
26-
private const int DefaultMaxRetries = 3;
27-
private const int DefaultMaxRetryDelay = 180_000;
28-
private const int DefaultExponentialBase = 5;
26+
private const int DefaultMaxRetries = 5;
27+
private const int DefaultMaxRetryDelay = 125_000;
28+
private const int DefaultExponentialBase = 2;
2929

3030
/// <summary>
3131
/// The number of data point to collect in batch.

0 commit comments

Comments
 (0)