Skip to content

Commit a73f256

Browse files
mrgrainmrgrain
authored
fix(logs): log retention custom resource timed out during deploy (#26995)
We use a custom resource to set the log retention for log groups created by the Lambda service. This custom resource handler code has a built-in retry mechanism to avoid throttling when executing many LogRetention CRs. Users can customize the number of possible retries, potentially retrying for a long time. This can cause the situation that further retries should be attempted, but the Lambda Function timeout is exceeded. The change sets the lambda execution timeout to its maximum value to allow for up to 15 minutes of retries. If the retry budget is exhausted, the handler will throw an error and exit early. Closes #24485 ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*
1 parent 1604cf4 commit a73f256

File tree

128 files changed

+3056
-5182
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

128 files changed

+3056
-5182
lines changed

‎packages/@aws-cdk-testing/framework-integ/test/aws-appsync/test/integ.js-resolver.js.snapshot/AppSyncJsResolverTestStack.assets.json

+6-6
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,15 @@
11
{
2-
"version": "33.0.0",
2+
"version": "34.0.0",
33
"files": {
4-
"4763e20569cc1d6f7ae496bbfb0b3e9bc205a1811c78c9a6bc18d949d737c2a9": {
4+
"a8515c042d9c942705087943220417be929ac44f968d8fcef2681681b400c0c0": {
55
"source": {
6-
"path": "asset.4763e20569cc1d6f7ae496bbfb0b3e9bc205a1811c78c9a6bc18d949d737c2a9",
6+
"path": "asset.a8515c042d9c942705087943220417be929ac44f968d8fcef2681681b400c0c0",
77
"packaging": "zip"
88
},
99
"destinations": {
1010
"current_account-current_region": {
1111
"bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
12-
"objectKey": "4763e20569cc1d6f7ae496bbfb0b3e9bc205a1811c78c9a6bc18d949d737c2a9.zip",
12+
"objectKey": "a8515c042d9c942705087943220417be929ac44f968d8fcef2681681b400c0c0.zip",
1313
"assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
1414
}
1515
}
@@ -53,15 +53,15 @@
5353
}
5454
}
5555
},
56-
"39bbb561d1400a68adab4e3b7ca0a4b5af2936645d1bd8ce29d72a4e7985c830": {
56+
"0351d32f030d2f7023c76f5a073f44cdcf81c0ddc63dae64a49d9d33269539ee": {
5757
"source": {
5858
"path": "AppSyncJsResolverTestStack.template.json",
5959
"packaging": "file"
6060
},
6161
"destinations": {
6262
"current_account-current_region": {
6363
"bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
64-
"objectKey": "39bbb561d1400a68adab4e3b7ca0a4b5af2936645d1bd8ce29d72a4e7985c830.json",
64+
"objectKey": "0351d32f030d2f7023c76f5a073f44cdcf81c0ddc63dae64a49d9d33269539ee.json",
6565
"assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
6666
}
6767
}

‎packages/@aws-cdk-testing/framework-integ/test/aws-appsync/test/integ.js-resolver.js.snapshot/AppSyncJsResolverTestStack.template.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -272,11 +272,12 @@
272272
"Properties": {
273273
"Handler": "index.handler",
274274
"Runtime": "nodejs18.x",
275+
"Timeout": 900,
275276
"Code": {
276277
"S3Bucket": {
277278
"Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"
278279
},
279-
"S3Key": "4763e20569cc1d6f7ae496bbfb0b3e9bc205a1811c78c9a6bc18d949d737c2a9.zip"
280+
"S3Key": "a8515c042d9c942705087943220417be929ac44f968d8fcef2681681b400c0c0.zip"
280281
},
281282
"Role": {
282283
"Fn::GetAtt": [

‎packages/@aws-cdk-testing/framework-integ/test/aws-appsync/test/integ.js-resolver.js.snapshot/JsResolverIntegTestDefaultTestDeployAssert57AD8D20.assets.json

+6-6
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,28 @@
11
{
2-
"version": "33.0.0",
2+
"version": "34.0.0",
33
"files": {
4-
"18f77fb224555ccad15f9c2e0f71ae6930fc011792c4f74ec74daaa2bbd9a33f": {
4+
"c7f4a8ca56d10961cf5e40d181faddf14f2ea8b39c5d65c7b61a366b17c7a2ce": {
55
"source": {
6-
"path": "asset.18f77fb224555ccad15f9c2e0f71ae6930fc011792c4f74ec74daaa2bbd9a33f.bundle",
6+
"path": "asset.c7f4a8ca56d10961cf5e40d181faddf14f2ea8b39c5d65c7b61a366b17c7a2ce.bundle",
77
"packaging": "zip"
88
},
99
"destinations": {
1010
"current_account-current_region": {
1111
"bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
12-
"objectKey": "18f77fb224555ccad15f9c2e0f71ae6930fc011792c4f74ec74daaa2bbd9a33f.zip",
12+
"objectKey": "c7f4a8ca56d10961cf5e40d181faddf14f2ea8b39c5d65c7b61a366b17c7a2ce.zip",
1313
"assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
1414
}
1515
}
1616
},
17-
"9b28bfafd748e7836cc59ca3e1167ed0048cf831b6df7efa11648ffa3c43e10e": {
17+
"763d169d10d22f0a01c5cab6482fd43062c1c41461ccb9546f484d0d8a4239bd": {
1818
"source": {
1919
"path": "JsResolverIntegTestDefaultTestDeployAssert57AD8D20.template.json",
2020
"packaging": "file"
2121
},
2222
"destinations": {
2323
"current_account-current_region": {
2424
"bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
25-
"objectKey": "9b28bfafd748e7836cc59ca3e1167ed0048cf831b6df7efa11648ffa3c43e10e.json",
25+
"objectKey": "763d169d10d22f0a01c5cab6482fd43062c1c41461ccb9546f484d0d8a4239bd.json",
2626
"assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
2727
}
2828
}

‎packages/@aws-cdk-testing/framework-integ/test/aws-appsync/test/integ.js-resolver.js.snapshot/JsResolverIntegTestDefaultTestDeployAssert57AD8D20.template.json

+3-3
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@
4949
"Payload.data.addTest.id",
5050
"Payload.data.addTest.id"
5151
],
52-
"salt": "1691091128259"
52+
"salt": "1693924790471"
5353
},
5454
"UpdateReplacePolicy": "Delete",
5555
"DeletionPolicy": "Delete"
@@ -158,7 +158,7 @@
158158
"S3Bucket": {
159159
"Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"
160160
},
161-
"S3Key": "18f77fb224555ccad15f9c2e0f71ae6930fc011792c4f74ec74daaa2bbd9a33f.zip"
161+
"S3Key": "c7f4a8ca56d10961cf5e40d181faddf14f2ea8b39c5d65c7b61a366b17c7a2ce.zip"
162162
},
163163
"Timeout": 120,
164164
"Handler": "index.handler",
@@ -226,7 +226,7 @@
226226
}
227227
},
228228
"flattenResponse": "false",
229-
"salt": "1691091128260"
229+
"salt": "1693924790503"
230230
},
231231
"UpdateReplacePolicy": "Delete",
232232
"DeletionPolicy": "Delete"

‎packages/@aws-cdk-testing/framework-integ/test/aws-appsync/test/integ.js-resolver.js.snapshot/asset.4763e20569cc1d6f7ae496bbfb0b3e9bc205a1811c78c9a6bc18d949d737c2a9/index.js

-187
This file was deleted.

‎packages/@aws-cdk-testing/framework-integ/test/aws-appsync/test/integ.js-resolver.js.snapshot/asset.4763e20569cc1d6f7ae496bbfb0b3e9bc205a1811c78c9a6bc18d949d737c2a9/index.d.ts renamed to ‎packages/@aws-cdk-testing/framework-integ/test/aws-appsync/test/integ.js-resolver.js.snapshot/asset.a8515c042d9c942705087943220417be929ac44f968d8fcef2681681b400c0c0/index.d.ts

File renamed without changes.

‎packages/@aws-cdk-testing/framework-integ/test/aws-appsync/test/integ.js-resolver.js.snapshot/asset.a8515c042d9c942705087943220417be929ac44f968d8fcef2681681b400c0c0/index.js

+192
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎packages/@aws-cdk/aws-neptune-alpha/test/integ.cluster.js.snapshot/asset.4763e20569cc1d6f7ae496bbfb0b3e9bc205a1811c78c9a6bc18d949d737c2a9/index.ts renamed to ‎packages/@aws-cdk-testing/framework-integ/test/aws-appsync/test/integ.js-resolver.js.snapshot/asset.a8515c042d9c942705087943220417be929ac44f968d8fcef2681681b400c0c0/index.ts

+9-3
Original file line numberDiff line numberDiff line change
@@ -91,11 +91,13 @@ export async function handler(event: LogRetentionEvent, context: AWSLambda.Conte
9191
const logGroupRegion = event.ResourceProperties.LogGroupRegion;
9292

9393
// Parse to AWS SDK retry options
94-
const withDelay = makeWithDelay(parseIntOptional(event.ResourceProperties.SdkRetry?.maxRetries));
94+
const maxRetries = parseIntOptional(event.ResourceProperties.SdkRetry?.maxRetries) ?? 5;
95+
const withDelay = makeWithDelay(maxRetries);
9596

9697
const sdkConfig: Logs.CloudWatchLogsClientConfig = {
9798
logger: console,
9899
region: logGroupRegion,
100+
maxAttempts: Math.max(5, maxRetries), // Use a minimum for SDK level retries, because it might include retryable failures that withDelay isn't checking for
99101
};
100102
const client = new Logs.CloudWatchLogsClient(sdkConfig);
101103

@@ -185,7 +187,7 @@ function parseIntOptional(value?: string, base = 10): number | undefined {
185187
}
186188

187189
function makeWithDelay(
188-
maxRetries: number = 5,
190+
maxRetries: number,
189191
delayBase: number = 100,
190192
delayCap = 10 * 1000, // 10s
191193
): (block: () => Promise<void>) => Promise<void> {
@@ -202,7 +204,11 @@ function makeWithDelay(
202204
try {
203205
return await block();
204206
} catch (error: any) {
205-
if (error instanceof Logs.OperationAbortedException || error.name === 'OperationAbortedException') {
207+
if (
208+
error instanceof Logs.OperationAbortedException
209+
|| error.name === 'OperationAbortedException'
210+
|| error.name === 'ThrottlingException' // There is no class to check with instanceof, see https://github.com/aws/aws-sdk-js-v3/issues/5140
211+
) {
206212
if (attempts < maxRetries ) {
207213
attempts++;
208214
await new Promise(resolve => setTimeout(resolve, calculateDelay(attempts, delayBase, delayCap)));

0 commit comments

Comments
 (0)