Skip to content

Commit 34950ad

Browse files
iliapolorix0rrr
iliapolo
and
rix0rrr
authored
feat(cli): configurable timeout for ECS hotswap operation (#417)
During hotswap deployment, we wait for the ECS service to become stable: https://github.com/aws/aws-cdk-cli/blob/f6bd7b9056186c00160ece09ac2d4ddc2ad72f35/packages/%40aws-cdk/tmp-toolkit-helpers/src/api/hotswap/ecs-services.ts#L153-L156 The timeout is hardcoded to 600 seconds, which is a very long time to **have** to wait. This PR add an additional property to the already existing ECS overrides bag, to configure this timeout. ### Notes Our integ tests currently suffer from this because they needlessly wait the hard-coded 10 minute timeout in a test that asserts ECS updates error handling. With this option, we set the operation timeout to 10 seconds and reduce the total integ suite duration **from 17min to 9min**. --- By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license --------- Co-authored-by: Rico Hermans <[email protected]>
1 parent dbf44a8 commit 34950ad

File tree

13 files changed

+268
-11
lines changed

13 files changed

+268
-11
lines changed

‎packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cdk-hotswap-deployment-for-ecs-service-detects-failed-deployment-and-errors.integtest.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ integTest(
1010

1111
// WHEN
1212
const deployOutput = await fixture.cdkDeploy('ecs-hotswap', {
13-
options: ['--hotswap'],
13+
options: ['--hotswap', '--hotswap-ecs-stabilization-timeout-seconds', '10'],
1414
modEnv: {
1515
USE_INVALID_ECS_HOTSWAP_IMAGE: 'true',
1616
},

‎packages/@aws-cdk/toolkit-lib/lib/api/aws-auth/sdk.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -488,7 +488,7 @@ export interface IECSClient {
488488
registerTaskDefinition(input: RegisterTaskDefinitionCommandInput): Promise<RegisterTaskDefinitionCommandOutput>;
489489
updateService(input: UpdateServiceCommandInput): Promise<UpdateServiceCommandOutput>;
490490
// Waiters
491-
waitUntilServicesStable(input: DescribeServicesCommandInput): Promise<WaiterResult>;
491+
waitUntilServicesStable(input: DescribeServicesCommandInput, timeoutSeconds?: number): Promise<WaiterResult>;
492492
}
493493

494494
export interface IElasticLoadBalancingV2Client {
@@ -827,11 +827,11 @@ export class SDK {
827827
updateService: (input: UpdateServiceCommandInput): Promise<UpdateServiceCommandOutput> =>
828828
client.send(new UpdateServiceCommand(input)),
829829
// Waiters
830-
waitUntilServicesStable: (input: DescribeServicesCommandInput): Promise<WaiterResult> => {
830+
waitUntilServicesStable: (input: DescribeServicesCommandInput, timeoutSeconds?: number): Promise<WaiterResult> => {
831831
return waitUntilServicesStable(
832832
{
833833
client,
834-
maxWaitTime: 600,
834+
maxWaitTime: timeoutSeconds ?? 600,
835835
minDelay: 6,
836836
maxDelay: 6,
837837
},

‎packages/@aws-cdk/toolkit-lib/lib/api/hotswap/common.ts

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,21 +86,27 @@ export class EcsHotswapProperties {
8686
readonly minimumHealthyPercent?: number;
8787
// The upper limit on the number of your service's tasks that are allowed in the RUNNING or PENDING state during a deployment, as a percentage of the desiredCount
8888
readonly maximumHealthyPercent?: number;
89+
// The number of seconds to wait for a single service to reach stable state.
90+
readonly stabilizationTimeoutSeconds?: number;
8991

90-
public constructor (minimumHealthyPercent?: number, maximumHealthyPercent?: number) {
92+
public constructor (minimumHealthyPercent?: number, maximumHealthyPercent?: number, stabilizationTimeoutSeconds?: number) {
9193
if (minimumHealthyPercent !== undefined && minimumHealthyPercent < 0 ) {
9294
throw new ToolkitError('hotswap-ecs-minimum-healthy-percent can\'t be a negative number');
9395
}
9496
if (maximumHealthyPercent !== undefined && maximumHealthyPercent < 0 ) {
9597
throw new ToolkitError('hotswap-ecs-maximum-healthy-percent can\'t be a negative number');
9698
}
99+
if (stabilizationTimeoutSeconds !== undefined && stabilizationTimeoutSeconds < 0 ) {
100+
throw new ToolkitError('hotswap-ecs-stabilization-timeout-seconds can\'t be a negative number');
101+
}
97102
// In order to preserve the current behaviour, when minimumHealthyPercent is not defined, it will be set to the currently default value of 0
98103
if (minimumHealthyPercent == undefined) {
99104
this.minimumHealthyPercent = 0;
100105
} else {
101106
this.minimumHealthyPercent = minimumHealthyPercent;
102107
}
103108
this.maximumHealthyPercent = maximumHealthyPercent;
109+
this.stabilizationTimeoutSeconds = stabilizationTimeoutSeconds;
104110
}
105111

106112
/**

‎packages/@aws-cdk/toolkit-lib/lib/api/hotswap/ecs-services.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,7 @@ export async function isHotswappableEcsServiceChange(
131131
let ecsHotswapProperties = hotswapPropertyOverrides.ecsHotswapProperties;
132132
let minimumHealthyPercent = ecsHotswapProperties?.minimumHealthyPercent;
133133
let maximumHealthyPercent = ecsHotswapProperties?.maximumHealthyPercent;
134+
let stabilizationTimeoutSeconds = ecsHotswapProperties?.stabilizationTimeoutSeconds;
134135

135136
// Step 2 - update the services using that TaskDefinition to point to the new TaskDefinition Revision
136137
// Forcing New Deployment and setting Minimum Healthy Percent to 0.
@@ -153,7 +154,7 @@ export async function isHotswappableEcsServiceChange(
153154
await sdk.ecs().waitUntilServicesStable({
154155
cluster: update.service?.clusterArn,
155156
services: [service.serviceArn],
156-
});
157+
}, stabilizationTimeoutSeconds);
157158
}),
158159
);
159160
},

‎packages/@aws-cdk/toolkit-lib/test/api/hotswap/ecs-services-hotswap-deployments.test.ts

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -724,5 +724,97 @@ describe.each([
724724
},
725725
forceNewDeployment: true,
726726
});
727+
expect(mockECSClient).toHaveReceivedCommandWith(DescribeServicesCommand, {
728+
cluster: 'arn:aws:ecs:region:account:service/my-cluster',
729+
services: ['arn:aws:ecs:region:account:service/my-cluster/my-service'],
730+
});
731+
});
732+
});
733+
734+
test.each([
735+
// default case
736+
[101, undefined],
737+
[2, 10],
738+
[11, 60],
739+
])('DesribeService is called %p times when timeout is %p', async (describeAttempts: number, timeoutSeconds?: number) => {
740+
setup.setCurrentCfnStackTemplate({
741+
Resources: {
742+
TaskDef: {
743+
Type: 'AWS::ECS::TaskDefinition',
744+
Properties: {
745+
Family: 'my-task-def',
746+
ContainerDefinitions: [
747+
{ Image: 'image1' },
748+
],
749+
},
750+
},
751+
Service: {
752+
Type: 'AWS::ECS::Service',
753+
Properties: {
754+
TaskDefinition: { Ref: 'TaskDef' },
755+
},
756+
},
757+
},
758+
});
759+
setup.pushStackResourceSummaries(
760+
setup.stackSummaryOf('Service', 'AWS::ECS::Service',
761+
'arn:aws:ecs:region:account:service/my-cluster/my-service'),
762+
);
763+
mockECSClient.on(RegisterTaskDefinitionCommand).resolves({
764+
taskDefinition: {
765+
taskDefinitionArn: 'arn:aws:ecs:region:account:task-definition/my-task-def:3',
766+
},
767+
});
768+
const cdkStackArtifact = setup.cdkStackArtifactOf({
769+
template: {
770+
Resources: {
771+
TaskDef: {
772+
Type: 'AWS::ECS::TaskDefinition',
773+
Properties: {
774+
Family: 'my-task-def',
775+
ContainerDefinitions: [
776+
{ Image: 'image2' },
777+
],
778+
},
779+
},
780+
Service: {
781+
Type: 'AWS::ECS::Service',
782+
Properties: {
783+
TaskDefinition: { Ref: 'TaskDef' },
784+
},
785+
},
786+
},
787+
},
788+
});
789+
790+
// WHEN
791+
let ecsHotswapProperties = new EcsHotswapProperties(undefined, undefined, timeoutSeconds);
792+
// mock the client such that the service never becomes stable using desiredCount > runningCount
793+
mockECSClient.on(DescribeServicesCommand).resolves({
794+
services: [
795+
{
796+
serviceArn: 'arn:aws:ecs:region:account:service/my-cluster/my-service',
797+
taskDefinition: 'arn:aws:ecs:region:account:task-definition/my-task-def:3',
798+
desiredCount: 1,
799+
runningCount: 0,
800+
},
801+
],
802+
});
803+
804+
jest.useFakeTimers();
805+
jest.spyOn(global, 'setTimeout').mockImplementation((callback, ms) => {
806+
callback();
807+
jest.advanceTimersByTime(ms ?? 0);
808+
return {} as NodeJS.Timeout;
727809
});
810+
811+
await expect(hotswapMockSdkProvider.tryHotswapDeployment(
812+
HotswapMode.HOTSWAP_ONLY,
813+
cdkStackArtifact,
814+
{},
815+
new HotswapPropertyOverrides(ecsHotswapProperties),
816+
)).rejects.toThrow('Resource is not in the expected state due to waiter status');
817+
818+
// THEN
819+
expect(mockECSClient).toHaveReceivedCommandTimes(DescribeServicesCommand, describeAttempts);
728820
});

‎packages/aws-cdk/README.md

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -505,19 +505,34 @@ Hotswapping is currently supported for the following changes
505505
- VTL mapping template changes for AppSync Resolvers and Functions.
506506
- Schema changes for AppSync GraphQL Apis.
507507

508-
You can optionally configure the behavior of your hotswap deployments in `cdk.json`. Currently you can only configure ECS hotswap behavior:
508+
You can optionally configure the behavior of your hotswap deployments. Currently you can only configure ECS hotswap behavior:
509+
510+
| Property | Description | Default |
511+
|--------------------------------|--------------------------------------|-------------|
512+
| minimumHealthyPercent | Lower limit on the number of your service's tasks that must remain in the RUNNING state during a deployment, as a percentage of the desiredCount | **REPLICA:** 100, **DAEMON:** 0 |
513+
| maximumHealthyPercent | Upper limit on the number of your service's tasks that are allowed in the RUNNING or PENDING state during a deployment, as a percentage of the desiredCount | **REPLICA:** 200, **DAEMON:**: N/A |
514+
| stabilizationTimeoutSeconds | Number of seconds to wait for a single service to reach stable state, where the desiredCount is equal to the runningCount | 600 |
515+
516+
##### cdk.json
509517

510518
```json
511519
{
512520
"hotswap": {
513521
"ecs": {
514522
"minimumHealthyPercent": 100,
515-
"maximumHealthyPercent": 250
523+
"maximumHealthyPercent": 250,
524+
"stabilizationTimeoutSeconds": 300,
516525
}
517526
}
518527
}
519528
```
520529

530+
##### cli arguments
531+
532+
```console
533+
cdk deploy --hotswap --hotswap-ecs-minimum-healthy-percent 100 --hotswap-ecs-maximum-healthy-percent 250 --hotswap-ecs-stabilization-timeout-seconds 300
534+
```
535+
521536
**⚠ Note #1**: This command deliberately introduces drift in CloudFormation stacks in order to speed up deployments.
522537
For this reason, only use it for development purposes.
523538
**Never use this flag for your production deployments**!

‎packages/aws-cdk/lib/cli/cdk-toolkit.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,8 @@ import * as uuid from 'uuid';
99
import { CliIoHost } from './io-host';
1010
import type { Configuration } from './user-configuration';
1111
import { PROJECT_CONFIG } from './user-configuration';
12+
import type { ToolkitAction } from '../../../@aws-cdk/toolkit-lib';
1213
import { StackSelectionStrategy, ToolkitError } from '../../../@aws-cdk/toolkit-lib';
13-
import type { ToolkitAction } from '../../../@aws-cdk/toolkit-lib/lib/api';
1414
import { asIoHelper } from '../../../@aws-cdk/toolkit-lib/lib/api/io/private';
1515
import { PermissionChangeType } from '../../../@aws-cdk/toolkit-lib/lib/payloads';
1616
import type { ToolkitOptions } from '../../../@aws-cdk/toolkit-lib/lib/toolkit';
@@ -390,6 +390,7 @@ export class CdkToolkit {
390390
hotswapPropertyOverrides.ecsHotswapProperties = new EcsHotswapProperties(
391391
hotswapPropertiesFromSettings.ecs?.minimumHealthyPercent,
392392
hotswapPropertiesFromSettings.ecs?.maximumHealthyPercent,
393+
hotswapPropertiesFromSettings.ecs?.stabilizationTimeoutSeconds,
393394
);
394395

395396
const stacks = stackCollection.stackArtifacts;

‎packages/aws-cdk/lib/cli/cli-config.ts

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -158,6 +158,18 @@ export async function makeConfig(): Promise<CliConfig> {
158158
'and falls back to a full deployment if that is not possible. ' +
159159
'Do not use this in production environments',
160160
},
161+
'hotswap-ecs-minimum-healthy-percent': {
162+
type: 'string',
163+
desc: 'Lower limit on the number of your service\'s tasks that must remain in the RUNNING state during a deployment, as a percentage of the desiredCount',
164+
},
165+
'hotswap-ecs-maximum-healthy-percent': {
166+
type: 'string',
167+
desc: 'Upper limit on the number of your service\'s tasks that are allowed in the RUNNING or PENDING state during a deployment, as a percentage of the desiredCount',
168+
},
169+
'hotswap-ecs-stabilization-timeout-seconds': {
170+
type: 'string',
171+
desc: 'Number of seconds to wait for a single service to reach stable state, where the desiredCount is equal to the runningCount',
172+
},
161173
'watch': {
162174
type: 'boolean',
163175
desc: 'Continuously observe the project files, ' +
@@ -275,6 +287,18 @@ export async function makeConfig(): Promise<CliConfig> {
275287
'which skips CloudFormation and updates the resources directly, ' +
276288
'and falls back to a full deployment if that is not possible.',
277289
},
290+
'hotswap-ecs-minimum-healthy-percent': {
291+
type: 'string',
292+
desc: 'Lower limit on the number of your service\'s tasks that must remain in the RUNNING state during a deployment, as a percentage of the desiredCount',
293+
},
294+
'hotswap-ecs-maximum-healthy-percent': {
295+
type: 'string',
296+
desc: 'Upper limit on the number of your service\'s tasks that are allowed in the RUNNING or PENDING state during a deployment, as a percentage of the desiredCount',
297+
},
298+
'hotswap-ecs-stabilization-timeout-seconds': {
299+
type: 'string',
300+
desc: 'Number of seconds to wait for a single service to reach stable state, where the desiredCount is equal to the runningCount',
301+
},
278302
'logs': {
279303
type: 'boolean',
280304
default: true,

‎packages/aws-cdk/lib/cli/convert-to-user-input.ts

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,9 @@ export function convertYargsToUserInput(args: any): UserInput {
114114
rollback: args.rollback,
115115
hotswap: args.hotswap,
116116
hotswapFallback: args.hotswapFallback,
117+
hotswapEcsMinimumHealthyPercent: args.hotswapEcsMinimumHealthyPercent,
118+
hotswapEcsMaximumHealthyPercent: args.hotswapEcsMaximumHealthyPercent,
119+
hotswapEcsStabilizationTimeoutSeconds: args.hotswapEcsStabilizationTimeoutSeconds,
117120
watch: args.watch,
118121
logs: args.logs,
119122
concurrency: args.concurrency,
@@ -159,6 +162,9 @@ export function convertYargsToUserInput(args: any): UserInput {
159162
rollback: args.rollback,
160163
hotswap: args.hotswap,
161164
hotswapFallback: args.hotswapFallback,
165+
hotswapEcsMinimumHealthyPercent: args.hotswapEcsMinimumHealthyPercent,
166+
hotswapEcsMaximumHealthyPercent: args.hotswapEcsMaximumHealthyPercent,
167+
hotswapEcsStabilizationTimeoutSeconds: args.hotswapEcsStabilizationTimeoutSeconds,
162168
logs: args.logs,
163169
concurrency: args.concurrency,
164170
STACKS: args.STACKS,
@@ -356,6 +362,9 @@ export function convertConfigToUserInput(config: any): UserInput {
356362
rollback: config.deploy?.rollback,
357363
hotswap: config.deploy?.hotswap,
358364
hotswapFallback: config.deploy?.hotswapFallback,
365+
hotswapEcsMinimumHealthyPercent: config.deploy?.hotswapEcsMinimumHealthyPercent,
366+
hotswapEcsMaximumHealthyPercent: config.deploy?.hotswapEcsMaximumHealthyPercent,
367+
hotswapEcsStabilizationTimeoutSeconds: config.deploy?.hotswapEcsStabilizationTimeoutSeconds,
359368
watch: config.deploy?.watch,
360369
logs: config.deploy?.logs,
361370
concurrency: config.deploy?.concurrency,
@@ -389,6 +398,9 @@ export function convertConfigToUserInput(config: any): UserInput {
389398
rollback: config.watch?.rollback,
390399
hotswap: config.watch?.hotswap,
391400
hotswapFallback: config.watch?.hotswapFallback,
401+
hotswapEcsMinimumHealthyPercent: config.watch?.hotswapEcsMinimumHealthyPercent,
402+
hotswapEcsMaximumHealthyPercent: config.watch?.hotswapEcsMaximumHealthyPercent,
403+
hotswapEcsStabilizationTimeoutSeconds: config.watch?.hotswapEcsStabilizationTimeoutSeconds,
392404
logs: config.watch?.logs,
393405
concurrency: config.watch?.concurrency,
394406
};

‎packages/aws-cdk/lib/cli/parse-command-line-arguments.ts

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -464,6 +464,21 @@ export function parseCommandLineArguments(args: Array<string>): any {
464464
type: 'boolean',
465465
desc: "Attempts to perform a 'hotswap' deployment, which skips CloudFormation and updates the resources directly, and falls back to a full deployment if that is not possible. Do not use this in production environments",
466466
})
467+
.option('hotswap-ecs-minimum-healthy-percent', {
468+
default: undefined,
469+
type: 'string',
470+
desc: "Lower limit on the number of your service's tasks that must remain in the RUNNING state during a deployment, as a percentage of the desiredCount",
471+
})
472+
.option('hotswap-ecs-maximum-healthy-percent', {
473+
default: undefined,
474+
type: 'string',
475+
desc: "Upper limit on the number of your service's tasks that are allowed in the RUNNING or PENDING state during a deployment, as a percentage of the desiredCount",
476+
})
477+
.option('hotswap-ecs-stabilization-timeout-seconds', {
478+
default: undefined,
479+
type: 'string',
480+
desc: 'Number of seconds to wait for a single service to reach stable state, where the desiredCount is equal to the runningCount',
481+
})
467482
.option('watch', {
468483
default: undefined,
469484
type: 'boolean',
@@ -628,6 +643,21 @@ export function parseCommandLineArguments(args: Array<string>): any {
628643
type: 'boolean',
629644
desc: "Attempts to perform a 'hotswap' deployment, which skips CloudFormation and updates the resources directly, and falls back to a full deployment if that is not possible.",
630645
})
646+
.option('hotswap-ecs-minimum-healthy-percent', {
647+
default: undefined,
648+
type: 'string',
649+
desc: "Lower limit on the number of your service's tasks that must remain in the RUNNING state during a deployment, as a percentage of the desiredCount",
650+
})
651+
.option('hotswap-ecs-maximum-healthy-percent', {
652+
default: undefined,
653+
type: 'string',
654+
desc: "Upper limit on the number of your service's tasks that are allowed in the RUNNING or PENDING state during a deployment, as a percentage of the desiredCount",
655+
})
656+
.option('hotswap-ecs-stabilization-timeout-seconds', {
657+
default: undefined,
658+
type: 'string',
659+
desc: 'Number of seconds to wait for a single service to reach stable state, where the desiredCount is equal to the runningCount',
660+
})
631661
.option('logs', {
632662
default: true,
633663
type: 'boolean',

‎packages/aws-cdk/lib/cli/user-configuration.ts

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -305,8 +305,9 @@ export function commandLineArgumentsToSettings(argv: Arguments): Settings {
305305
ignoreNoStacks: argv['ignore-no-stacks'],
306306
hotswap: {
307307
ecs: {
308-
minimumEcsHealthyPercent: argv.minimumEcsHealthyPercent,
309-
maximumEcsHealthyPercent: argv.maximumEcsHealthyPercent,
308+
minimumHealthyPercent: argv.hotswapEcsMinimumHealthyPercent,
309+
maximumHealthyPercent: argv.hotswapEcsMaximumHealthyPercent,
310+
stabilizationTimeoutSeconds: argv.hotswapEcsStabilizationTimeoutSeconds,
310311
},
311312
},
312313
unstable: argv.unstable,

0 commit comments

Comments
 (0)