Skip to content

Commit 8e527ce

Browse files
SergeyRyabininSergeyRyabinin
committed
S3 CRT retry strategy config
1 parent cc33599 commit 8e527ce

File tree

4 files changed

+146
-56
lines changed

4 files changed

+146
-56
lines changed

‎generated/src/aws-cpp-sdk-s3-crt/include/aws/s3-crt/ClientConfiguration.h

Lines changed: 2 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -5,67 +5,13 @@
55

66
#pragma once
77

8-
#include <aws/s3-crt/S3Crt_EXPORTS.h>
98
#include <aws/s3-crt/S3CrtClientConfiguration.h>
10-
#include <aws/core/utils/memory/stl/AWSString.h>
11-
#include <aws/core/http/Scheme.h>
12-
#include <aws/core/Region.h>
13-
#include <aws/crt/io/Bootstrap.h>
14-
#include <aws/crt/io/TlsOptions.h>
159

1610
namespace Aws
1711
{
1812
namespace S3Crt
1913
{
20-
struct AWS_S3CRT_API ClientConfiguration : Aws::S3Crt::S3CrtClientConfiguration
21-
{
22-
ClientConfiguration() : Aws::S3Crt::S3CrtClientConfiguration(),
23-
partSize(5 * 1024 * 1024),
24-
throughputTargetGbps(2.0) {}
25-
26-
ClientConfiguration(const S3Crt::ClientConfiguration& other,
27-
Client::AWSAuthV4Signer::PayloadSigningPolicy iPayloadSigningPolicy = Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
28-
bool iUseVirtualAddressing = true,
29-
US_EAST_1_REGIONAL_ENDPOINT_OPTION iUseUSEast1RegionalEndPointOption = US_EAST_1_REGIONAL_ENDPOINT_OPTION::NOT_SET):
30-
S3CrtClientConfiguration(other, iPayloadSigningPolicy,iUseVirtualAddressing, iUseUSEast1RegionalEndPointOption),
31-
clientBootstrap(other.clientBootstrap),
32-
partSize(other.partSize),
33-
tlsConnectionOptions(other.tlsConnectionOptions),
34-
throughputTargetGbps(other.throughputTargetGbps),
35-
downloadMemoryUsageWindow(other.downloadMemoryUsageWindow),
36-
clientShutdownCallback(other.clientShutdownCallback),
37-
shutdownCallbackUserData(other.shutdownCallbackUserData)
38-
{}
39-
40-
ClientConfiguration& operator=(const ClientConfiguration& other) = default;
41-
42-
/** Client bootstrap used for common staples such as event loop group, host resolver, etc..
43-
* If this is nullptr, SDK will create a default one for you.
44-
*/
45-
std::shared_ptr<Aws::Crt::Io::ClientBootstrap> clientBootstrap;
46-
47-
/** Size of parts the files will be downloaded or uploaded in. Useful for Put/GetObject APIs
48-
* defaults to 5MB, if user set it to be less than 5MB, SDK will set it to 5MB.
49-
*/
50-
size_t partSize = 5 * 1024 * 1024;
51-
52-
/** TLS Options to be used for each connection.
53-
* If scheme is Https and tlsConnectionOptions is null, SDK will create a default one for you.
54-
*/
55-
std::shared_ptr<Aws::Crt::Io::TlsConnectionOptions> tlsConnectionOptions;
56-
57-
/* Throughput target in Gbps that we are trying to reach. Normally it's the NIC's throughput */
58-
double throughputTargetGbps = 2.0;
59-
60-
/** Control the maximum memory used by downloads.
61-
* When set to a value > 0, the SDK uses flow control to bring the memory usage very close
62-
* to the specified window. Without this cap, memory usage grows proportional to file size.
63-
*/
64-
size_t downloadMemoryUsageWindow = 0;
65-
66-
/* Callback and associated user data for when the client has completed its shutdown process. */
67-
std::function<void(void*)> clientShutdownCallback;
68-
void *shutdownCallbackUserData = nullptr;
69-
};
14+
/* Backward compatibility, please use just S3CrtClientConfiguration */
15+
using ClientConfiguration = Aws::S3Crt::S3CrtClientConfiguration;
7016
}
7117
}

‎generated/src/aws-cpp-sdk-s3-crt/include/aws/s3-crt/S3CrtClientConfiguration.h

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,9 @@
99
#include <aws/s3-crt/S3ExpressIdentityProvider.h>
1010
#include <aws/core/client/GenericClientConfiguration.h>
1111
#include <aws/core/auth/signer/AWSAuthV4Signer.h>
12+
#include <aws/crt/io/Bootstrap.h>
13+
#include <aws/crt/io/TlsOptions.h>
14+
#include <aws/io/retry_strategy.h>
1215

1316

1417
namespace Aws
@@ -62,6 +65,80 @@ namespace Aws
6265
IdentityProviderSupplier identityProviderSupplier = [](const S3CrtClient &client) -> std::shared_ptr<S3ExpressIdentityProvider> {
6366
return Aws::MakeShared<DefaultS3ExpressIdentityProvider>("S3CrtClientConfiguration", client);
6467
};
68+
69+
/* S3 CRT specifics */
70+
S3CrtClientConfiguration(const S3CrtClientConfiguration&) = default;
71+
S3CrtClientConfiguration& operator=(const S3CrtClientConfiguration&) = default;
72+
73+
struct S3CrtConfigFactories
74+
{
75+
/**
76+
* CRT retry strategy factory method.
77+
* Default will try to match SDK Retry Stratgey (which defaults to DefaultRetryStrategy)
78+
* Note: CRT retry strategy is not SDK retry strategy, retry algorithm and request counting are different.
79+
*/
80+
std::function<aws_retry_strategy*(const S3Crt::S3CrtClientConfiguration&)> retryStrategyCreateFn;
81+
82+
static S3CrtConfigFactories defaultFactories;
83+
};
84+
85+
S3CrtConfigFactories crtConfigFactories = S3CrtConfigFactories::defaultFactories;
86+
87+
/** Client bootstrap used for common staples such as event loop group, host resolver, etc..
88+
* If this is nullptr, SDK will create a default one for you.
89+
*/
90+
std::shared_ptr<Aws::Crt::Io::ClientBootstrap> clientBootstrap;
91+
92+
/** Size of parts the files will be downloaded or uploaded in. Useful for Put/GetObject APIs
93+
* defaults to 5MB, if user set it to be less than 5MB, SDK will set it to 5MB.
94+
*/
95+
size_t partSize = 5 * 1024 * 1024;
96+
97+
/** TLS Options to be used for each connection.
98+
* If scheme is Https and tlsConnectionOptions is null, SDK will create a default one for you.
99+
*/
100+
std::shared_ptr<Aws::Crt::Io::TlsConnectionOptions> tlsConnectionOptions;
101+
102+
/* Throughput target in Gbps that we are trying to reach. Normally it's the NIC's throughput */
103+
double throughputTargetGbps = 2.0;
104+
105+
/** Control the maximum memory used by downloads.
106+
* When set to a value > 0, the SDK uses flow control to bring the memory usage very close
107+
* to the specified window. Without this cap, memory usage grows proportional to file size.
108+
*/
109+
size_t downloadMemoryUsageWindow = 0;
110+
111+
/* Callback and associated user data for when the client has completed its shutdown process. */
112+
std::function<void(void*)> clientShutdownCallback;
113+
void *shutdownCallbackUserData = nullptr;
114+
115+
struct CrtRetryStrategyConfig
116+
{
117+
enum class CrtRetryStrategyType
118+
{
119+
NOT_SET=0, /* Defaults to match SDK retry strategy. */
120+
DEFAULT,
121+
STANDARD,
122+
EXPONENTIAL_BACKOFF,
123+
NO_RETRY=-1,
124+
} crtRetryStrategyType = CrtRetryStrategyType::NOT_SET;
125+
126+
struct ExponentialBackoffConfig
127+
{
128+
/**
129+
* EXPONENTIAL_BACKOFF strategy config
130+
*/
131+
/* Maximum number of request retries. Specifying 0 here will fall back to the default set by aws-c-io. */
132+
size_t maxRetries = 0;
133+
134+
/* Scale factor (in milliseconds) for the back-off value. */
135+
uint32_t scaleFactorMs = 500;
136+
137+
/* Maximum delay between retries (in seconds). Specifying 0 here will fall back to the default set by aws-c-io. */
138+
uint32_t maxBackoffSecs = 20;
139+
} config;
140+
} crtRetryStrategyConfig;
141+
/* End of S3 CRT specifics */
65142
private:
66143
void LoadS3CrtSpecificConfig(const Aws::String& profileName);
67144
};

‎generated/src/aws-cpp-sdk-s3-crt/source/S3CrtClient.cpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -348,6 +348,10 @@ void S3CrtClient::init(const S3Crt::ClientConfiguration& config,
348348
Aws::Crt::Io::ClientBootstrap* clientBootstrap = config.clientBootstrap ? config.clientBootstrap.get() : Aws::GetDefaultClientBootstrap();
349349
s3CrtConfig.client_bootstrap = clientBootstrap->GetUnderlyingHandle();
350350

351+
using CrtRetryStrategyUniquePtr = std::unique_ptr<aws_retry_strategy, decltype(&aws_retry_strategy_release)>;
352+
auto retry_strategy = CrtRetryStrategyUniquePtr(config.crtConfigFactories.retryStrategyCreateFn(config), aws_retry_strategy_release);
353+
s3CrtConfig.retry_strategy = retry_strategy.get();
354+
351355
m_crtCredProvider = Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderDelegate({
352356
std::bind([](const std::shared_ptr<AWSCredentialsProvider>& provider) {
353357
if (provider == nullptr) {

‎generated/src/aws-cpp-sdk-s3-crt/source/S3CrtClientConfiguration.cpp

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44
*/
55

66
#include <aws/s3-crt/S3CrtClientConfiguration.h>
7+
#include <aws/core/Globals.h>
8+
#include <aws/core/client/RetryStrategy.h>
79

810
namespace Aws
911
{
@@ -97,6 +99,67 @@ S3CrtClientConfiguration::S3CrtClientConfiguration(const Client::ClientConfigura
9799
LoadS3CrtSpecificConfig(this->profileName);
98100
}
99101

102+
/* S3 CRT specifics */
103+
S3CrtClientConfiguration::S3CrtConfigFactories S3CrtClientConfiguration::S3CrtConfigFactories::defaultFactories = []()
104+
{
105+
S3CrtConfigFactories factories;
106+
107+
factories.retryStrategyCreateFn = [](const S3Crt::S3CrtClientConfiguration& config) -> aws_retry_strategy*
108+
{
109+
aws_event_loop_group *el_group = config.clientBootstrap ?
110+
config.clientBootstrap->GetUnderlyingHandle()->event_loop_group : Aws::GetDefaultClientBootstrap()->GetUnderlyingHandle()->event_loop_group;
111+
112+
using CrtRetryStrategyType = S3Crt::S3CrtClientConfiguration::CrtRetryStrategyConfig::CrtRetryStrategyType;
113+
114+
CrtRetryStrategyType strategyToUse = config.crtRetryStrategyConfig.crtRetryStrategyType;
115+
116+
if (strategyToUse == CrtRetryStrategyType::NOT_SET && config.retryStrategy)
117+
{
118+
if (config.retryStrategy)
119+
{
120+
if(Aws::String("standard") == config.retryStrategy->GetStrategyName())
121+
strategyToUse = CrtRetryStrategyType::STANDARD;
122+
else if(Aws::String("adaptive") == config.retryStrategy->GetStrategyName())
123+
strategyToUse = CrtRetryStrategyType::EXPONENTIAL_BACKOFF;
124+
else
125+
strategyToUse = CrtRetryStrategyType::DEFAULT;
126+
}
127+
}
128+
129+
if (strategyToUse == CrtRetryStrategyType::DEFAULT)
130+
{
131+
return nullptr;
132+
}
133+
134+
if (strategyToUse == CrtRetryStrategyType::STANDARD)
135+
{
136+
aws_standard_retry_options options {};
137+
return aws_retry_strategy_new_standard(Aws::get_aws_allocator(), &options);
138+
}
139+
140+
if (strategyToUse == CrtRetryStrategyType::EXPONENTIAL_BACKOFF)
141+
{
142+
aws_exponential_backoff_retry_options options {};
143+
options.el_group = el_group,
144+
options.max_retries = config.crtRetryStrategyConfig.config.maxRetries;
145+
options.backoff_scale_factor_ms = config.crtRetryStrategyConfig.config.scaleFactorMs;
146+
options.max_backoff_secs = config.crtRetryStrategyConfig.config.maxBackoffSecs;
147+
options.jitter_mode = AWS_EXPONENTIAL_BACKOFF_JITTER_FULL;
148+
return aws_retry_strategy_new_exponential_backoff(Aws::get_aws_allocator(), &options);
149+
}
150+
151+
if (strategyToUse == CrtRetryStrategyType::NO_RETRY)
152+
{
153+
aws_standard_retry_options options {};
154+
options.initial_bucket_capacity = 1;
155+
return aws_retry_strategy_new_standard(Aws::get_aws_allocator(), &options);
156+
}
157+
158+
return nullptr;
159+
};
100160

161+
return factories;
162+
}();
163+
/* End of S3 CRT specifics */
101164
} // namespace S3Crt
102165
} // namespace Aws

0 commit comments

Comments
 (0)