Skip to content

Commit c5cab08

Browse files
npalmnpalm
authored
Merge branch 'main' into dependabot/npm_and_yarn/lambdas/jest-mock-extended-3.0.7
2 parents da3f5f1 + 2e094cf commit c5cab08

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

48 files changed

+1138
-948
lines changed

‎.github/workflows/stale.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ jobs:
1818
stale-pr-message: >
1919
This pull request has been automatically marked as stale because it has not had activity in the last 30 days. It will be closed if no further activity occurs. Thank you for your contributions.
2020
21-
days-before-stale: 30
22-
days-before-close: 10
21+
days-before-stale: 90
22+
days-before-close: 14
2323
close-issue-label: "abandoned"
2424
exempt-issue-labels: "stale:exempt"

‎CHANGELOG.md

+15
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,20 @@
11
# Changelog
22

3+
## [5.11.0](https://github.com/philips-labs/terraform-aws-github-runner/compare/v5.10.4...v5.11.0) (2024-05-22)
4+
5+
6+
### Features
7+
8+
* add variable to configure ebs optimization for runner instances ([479b779](https://github.com/philips-labs/terraform-aws-github-runner/commit/479b779a71c77a62dd28d247f8a74cb75ce083f0))
9+
* add variable to configure ebs optimization for runner instances ([#3901](https://github.com/philips-labs/terraform-aws-github-runner/issues/3901)) ([479b779](https://github.com/philips-labs/terraform-aws-github-runner/commit/479b779a71c77a62dd28d247f8a74cb75ce083f0))
10+
* Restrict instance SSM permissions ([#3918](https://github.com/philips-labs/terraform-aws-github-runner/issues/3918)) ([9399cf2](https://github.com/philips-labs/terraform-aws-github-runner/commit/9399cf29bec963dfa305f367f37c098a76130371))
11+
12+
13+
### Bug Fixes
14+
15+
* adding missing permissions to boundaries ([#3873](https://github.com/philips-labs/terraform-aws-github-runner/issues/3873)) ([93e8d27](https://github.com/philips-labs/terraform-aws-github-runner/commit/93e8d2746b647539212dbc65887ec748a1d734b7))
16+
* **lambda:** bump the aws group across 1 directory with 6 updates ([#3907](https://github.com/philips-labs/terraform-aws-github-runner/issues/3907)) ([50dda9a](https://github.com/philips-labs/terraform-aws-github-runner/commit/50dda9a465229bdb8d106e7ebc5d5b1de115a286))
17+
318
## [5.10.4](https://github.com/philips-labs/terraform-aws-github-runner/compare/v5.10.3...v5.10.4) (2024-05-06)
419

520

‎README.md

+2
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,7 @@ Talk to the forestkeepers in the `runners-channel` on Slack.
179179
| <a name="input_log_level"></a> [log\_level](#input\_log\_level) | Logging level for lambda logging. Valid values are 'silly', 'trace', 'debug', 'info', 'warn', 'error', 'fatal'. | `string` | `"info"` | no |
180180
| <a name="input_logging_kms_key_id"></a> [logging\_kms\_key\_id](#input\_logging\_kms\_key\_id) | Specifies the kms key id to encrypt the logs with. | `string` | `null` | no |
181181
| <a name="input_logging_retention_in_days"></a> [logging\_retention\_in\_days](#input\_logging\_retention\_in\_days) | Specifies the number of days you want to retain log events for the lambda log group. Possible values are: 0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, and 3653. | `number` | `180` | no |
182+
| <a name="input_matcher_config_parameter_store_tier"></a> [matcher\_config\_parameter\_store\_tier](#input\_matcher\_config\_parameter\_store\_tier) | The tier of the parameter store for the matcher configuration. Valid values are `Standard`, and `Advanced`. | `string` | `"Standard"` | no |
182183
| <a name="input_metrics_namespace"></a> [metrics\_namespace](#input\_metrics\_namespace) | The namespace for the metrics created by the module. Merics will only be created if explicit enabled. | `string` | `"GitHub Runners"` | no |
183184
| <a name="input_minimum_running_time_in_minutes"></a> [minimum\_running\_time\_in\_minutes](#input\_minimum\_running\_time\_in\_minutes) | The time an ec2 action runner should be running at minimum before terminated, if not busy. | `number` | `null` | no |
184185
| <a name="input_pool_config"></a> [pool\_config](#input\_pool\_config) | The configuration for updating the pool. The `pool_size` to adjust to by the events triggered by the `schedule_expression`. For example you can configure a cron expression for weekdays to adjust the pool to 10 and another expression for the weekend to adjust the pool to 1. | <pre>list(object({<br> schedule_expression = string<br> size = number<br> }))</pre> | `[]` | no |
@@ -214,6 +215,7 @@ Talk to the forestkeepers in the `runners-channel` on Slack.
214215
| <a name="input_runner_name_prefix"></a> [runner\_name\_prefix](#input\_runner\_name\_prefix) | The prefix used for the GitHub runner name. The prefix will be used in the default start script to prefix the instance name when register the runner in GitHub. The value is availabe via an EC2 tag 'ghr:runner\_name\_prefix'. | `string` | `""` | no |
215216
| <a name="input_runner_os"></a> [runner\_os](#input\_runner\_os) | The EC2 Operating System type to use for action runner instances (linux,windows). | `string` | `"linux"` | no |
216217
| <a name="input_runner_run_as"></a> [runner\_run\_as](#input\_runner\_run\_as) | Run the GitHub actions agent as user. | `string` | `"ec2-user"` | no |
218+
| <a name="input_runners_ebs_optimized"></a> [runners\_ebs\_optimized](#input\_runners\_ebs\_optimized) | Enable EBS optimization for the runner instances. | `bool` | `false` | no |
217219
| <a name="input_runners_lambda_s3_key"></a> [runners\_lambda\_s3\_key](#input\_runners\_lambda\_s3\_key) | S3 key for runners lambda function. Required if using S3 bucket to specify lambdas. | `string` | `null` | no |
218220
| <a name="input_runners_lambda_s3_object_version"></a> [runners\_lambda\_s3\_object\_version](#input\_runners\_lambda\_s3\_object\_version) | S3 object version for runners lambda function. Useful if S3 versioning is enabled on source bucket. | `string` | `null` | no |
219221
| <a name="input_runners_lambda_zip"></a> [runners\_lambda\_zip](#input\_runners\_lambda\_zip) | File location of the lambda zip file for scaling runners. | `string` | `null` | no |

‎lambdas/functions/ami-housekeeper/package.json

+2-2
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,8 @@
3737
"ts-node-dev": "^2.0.0"
3838
},
3939
"dependencies": {
40-
"@aws-sdk/client-ec2": "^3.575.0",
41-
"@aws-sdk/client-ssm": "^3.575.0",
40+
"@aws-sdk/client-ec2": "^3.604.0",
41+
"@aws-sdk/client-ssm": "^3.603.0",
4242
"@aws-sdk/types": "^3.433.0",
4343
"@terraform-aws-github-runner/aws-powertools-util": "*",
4444
"@terraform-aws-github-runner/aws-ssm-util": "*",

‎lambdas/functions/control-plane/package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@
3838
"ts-node-dev": "^2.0.0"
3939
},
4040
"dependencies": {
41-
"@aws-sdk/client-ec2": "^3.575.0",
41+
"@aws-sdk/client-ec2": "^3.604.0",
4242
"@aws-sdk/types": "^3.433.0",
4343
"@middy/core": "^4.7.0",
4444
"@octokit/auth-app": "6.0.3",

‎lambdas/functions/control-plane/src/aws/runners.ts

+1
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,7 @@ function getRunnerInfo(runningInstances: DescribeInstancesResult) {
9494
}
9595

9696
export async function terminateRunner(instanceId: string): Promise<void> {
97+
logger.info(`Runner '${instanceId}' will be terminated.`);
9798
const ec2 = getTracedAWSV3Client(new EC2Client({ region: process.env.AWS_REGION }));
9899
await ec2.send(new TerminateInstancesCommand({ InstanceIds: [instanceId] }));
99100
logger.info(`Runner ${instanceId} has been terminated.`);

‎lambdas/functions/control-plane/src/scale-runners/scale-down.ts

+11-2
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,8 @@ async function listGitHubRunners(runner: RunnerInfo): Promise<GhRunners> {
8888
per_page: 100,
8989
});
9090
githubCache.runners.set(key, runners);
91-
91+
logger.debug(`[listGithubRunners] Cache set for ${key}`);
92+
logger.debug(`[listGithubRunners] Runners: ${JSON.stringify(runners)}`);
9293
return runners;
9394
}
9495

@@ -156,18 +157,25 @@ async function evaluateAndRemoveRunners(
156157
.filter((runner) => runner.owner === ownerTag)
157158
.sort(evictionStrategy === 'oldest_first' ? oldestFirstStrategy : newestFirstStrategy);
158159
logger.debug(`Found: '${ec2RunnersFiltered.length}' active GitHub runners with owner tag: '${ownerTag}'`);
160+
logger.debug(`Active GitHub runners with owner tag: '${ownerTag}': ${JSON.stringify(ec2RunnersFiltered)}`);
159161
for (const ec2Runner of ec2RunnersFiltered) {
160162
const ghRunners = await listGitHubRunners(ec2Runner);
161163
const ghRunnersFiltered = ghRunners.filter((runner: { name: string }) =>
162164
runner.name.endsWith(ec2Runner.instanceId),
163165
);
166+
logger.debug(
167+
`Found: '${ghRunnersFiltered.length}' GitHub runners for AWS runner instance: '${ec2Runner.instanceId}'`,
168+
);
169+
logger.debug(
170+
`GitHub runners for AWS runner instance: '${ec2Runner.instanceId}': ${JSON.stringify(ghRunnersFiltered)}`,
171+
);
164172
if (ghRunnersFiltered.length) {
165173
if (runnerMinimumTimeExceeded(ec2Runner)) {
166174
if (idleCounter > 0) {
167175
idleCounter--;
168176
logger.info(`Runner '${ec2Runner.instanceId}' will be kept idle.`);
169177
} else {
170-
logger.info(`Runner '${ec2Runner.instanceId}' will be terminated.`);
178+
logger.info(`Will try to terminate runners that are not busy`);
171179
await removeRunner(
172180
ec2Runner,
173181
ghRunnersFiltered.map((runner: { id: number }) => runner.id),
@@ -224,6 +232,7 @@ export async function scaleDown(): Promise<void> {
224232
const ec2Runners = await listRunners(environment);
225233
const activeEc2RunnersCount = ec2Runners.length;
226234
logger.info(`Found: '${activeEc2RunnersCount}' active GitHub EC2 runner instances before clean-up.`);
235+
logger.debug(`Active GitHub EC2 runner instances: ${JSON.stringify(ec2Runners)}`);
227236

228237
if (activeEc2RunnersCount === 0) {
229238
logger.debug(`No active runners found for environment: '${environment}'`);

‎lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts

+30
Original file line numberDiff line numberDiff line change
@@ -338,6 +338,12 @@ describe('scaleUp with GHES', () => {
338338
Name: '/github-action-runners/default/runners/config/i-12345',
339339
Value: 'TEST_JIT_CONFIG_ORG',
340340
Type: 'SecureString',
341+
Tags: [
342+
{
343+
Key: 'InstanceId',
344+
Value: 'i-12345',
345+
},
346+
],
341347
});
342348
});
343349

@@ -353,6 +359,12 @@ describe('scaleUp with GHES', () => {
353359
'--url https://github.enterprise.something/Codertocat --token 1234abcd ' +
354360
'--labels label1,label2 --runnergroup Default',
355361
Type: 'SecureString',
362+
Tags: [
363+
{
364+
Key: 'InstanceId',
365+
Value: 'i-12345',
366+
},
367+
],
356368
});
357369
});
358370
it.each(RUNNER_TYPES)(
@@ -708,6 +720,12 @@ describe('scaleUp with public GH', () => {
708720
Name: '/github-action-runners/default/runners/config/i-12345',
709721
Value: 'TEST_JIT_CONFIG_REPO',
710722
Type: 'SecureString',
723+
Tags: [
724+
{
725+
Key: 'InstanceId',
726+
Value: 'i-12345',
727+
},
728+
],
711729
});
712730
});
713731

@@ -724,6 +742,12 @@ describe('scaleUp with public GH', () => {
724742
Name: '/github-action-runners/default/runners/config/i-12345',
725743
Value: '--url https://github.com/Codertocat/hello-world --token 1234abcd --ephemeral',
726744
Type: 'SecureString',
745+
Tags: [
746+
{
747+
Key: 'InstanceId',
748+
Value: 'i-12345',
749+
},
750+
],
727751
});
728752
});
729753

@@ -741,6 +765,12 @@ describe('scaleUp with public GH', () => {
741765
Name: '/github-action-runners/default/runners/config/i-12345',
742766
Value: '--url https://github.com/Codertocat/hello-world --token 1234abcd --labels jit',
743767
Type: 'SecureString',
768+
Tags: [
769+
{
770+
Key: 'InstanceId',
771+
Value: 'i-12345',
772+
},
773+
],
744774
});
745775
});
746776

‎lambdas/functions/control-plane/src/scale-runners/scale-up.ts

+6-2
Original file line numberDiff line numberDiff line change
@@ -362,7 +362,9 @@ async function createRegistrationTokenConfig(
362362
});
363363

364364
for (const instance of instances) {
365-
await putParameter(`${githubRunnerConfig.ssmTokenPath}/${instance}`, runnerServiceConfig.join(' '), true);
365+
await putParameter(`${githubRunnerConfig.ssmTokenPath}/${instance}`, runnerServiceConfig.join(' '), true, {
366+
tags: [{ Key: 'InstanceId', Value: instance }],
367+
});
366368
if (isDelay) {
367369
// Delay to prevent AWS ssm rate limits by being within the max throughput limit
368370
await delay(25);
@@ -405,7 +407,9 @@ async function createJitConfig(githubRunnerConfig: CreateGitHubRunnerConfig, ins
405407
logger.debug('Runner JIT config for ephemeral runner generated.', {
406408
instance: instance,
407409
});
408-
await putParameter(`${githubRunnerConfig.ssmTokenPath}/${instance}`, runnerConfig.data.encoded_jit_config, true);
410+
await putParameter(`${githubRunnerConfig.ssmTokenPath}/${instance}`, runnerConfig.data.encoded_jit_config, true, {
411+
tags: [{ Key: 'InstanceId', Value: instance }],
412+
});
409413
if (isDelay) {
410414
// Delay to prevent AWS ssm rate limits by being within the max throughput limit
411415
await delay(25);

‎lambdas/functions/gh-agent-syncer/package.json

+2-2
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,8 @@
3737
"typescript": "^5.4.5"
3838
},
3939
"dependencies": {
40-
"@aws-sdk/client-s3": "^3.575.0",
41-
"@aws-sdk/lib-storage": "^3.575.0",
40+
"@aws-sdk/client-s3": "^3.600.0",
41+
"@aws-sdk/lib-storage": "^3.600.0",
4242
"@aws-sdk/types": "^3.433.0",
4343
"@middy/core": "^4.7.0",
4444
"@terraform-aws-github-runner/aws-powertools-util": "*",

‎lambdas/functions/termination-watcher/package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@
3535
"ts-node-dev": "^2.0.0"
3636
},
3737
"dependencies": {
38-
"@aws-sdk/client-ec2": "^3.575.0",
38+
"@aws-sdk/client-ec2": "^3.604.0",
3939
"@aws-sdk/types": "^3.433.0",
4040
"@middy/core": "^4.7.0",
4141
"@terraform-aws-github-runner/aws-powertools-util": "*",

‎lambdas/functions/webhook/package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@
3838
"typescript": "^5.4.5"
3939
},
4040
"dependencies": {
41-
"@aws-sdk/client-sqs": "^3.575.0",
41+
"@aws-sdk/client-sqs": "^3.600.0",
4242
"@middy/core": "^4.7.0",
4343
"@octokit/rest": "^20.0.1",
4444
"@octokit/types": "^12.5.0",

‎lambdas/functions/webhook/src/ConfigResolver.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ export class Config {
2929
Config.matcherConfig = JSON.parse(matcherConfigVal) as Array<RunnerMatcherConfig>;
3030
logger.debug('Loaded queues config', { matcherConfig: Config.matcherConfig });
3131
}
32-
const workflowJobEventSecondaryQueue = process.env.SQS_WORKFLOW_JOB_QUEUE ?? undefined;
32+
const workflowJobEventSecondaryQueue = process.env.SQS_WORKFLOW_JOB_QUEUE || undefined;
3333
return new Config(repositoryAllowList, workflowJobEventSecondaryQueue);
3434
}
3535

‎lambdas/functions/webhook/src/sqs/index.test.ts

+16-2
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@ jest.mock('@aws-sdk/client-sqs', () => ({
1616
}));
1717
jest.mock('@terraform-aws-github-runner/aws-ssm-util');
1818

19+
import { SQS } from '@aws-sdk/client-sqs';
20+
1921
describe('Test sending message to SQS.', () => {
2022
const queueUrl = 'https://sqs.eu-west-1.amazonaws.com/123456789/queued-builds';
2123
const message = {
@@ -98,15 +100,27 @@ describe('Test sending message to SQS.', () => {
98100
expect(result).resolves;
99101
});
100102

101-
it('Does not send webhook events to workflow job event copy queue', async () => {
103+
it('Does not send webhook events to workflow job event copy queue when job queue is not in environment', async () => {
104+
// Arrange
105+
delete process.env.SQS_WORKFLOW_JOB_QUEUE;
106+
const config = await Config.load();
107+
108+
// Act
109+
await sendWebhookEventToWorkflowJobQueue(message, config);
110+
111+
// Assert
112+
expect(SQS).not.toHaveBeenCalled();
113+
});
114+
115+
it('Does not send webhook events to workflow job event copy queue when job queue is set to empty string', async () => {
102116
// Arrange
103117
process.env.SQS_WORKFLOW_JOB_QUEUE = '';
104118
const config = await Config.load();
105119
// Act
106120
await sendWebhookEventToWorkflowJobQueue(message, config);
107121

108122
// Assert
109-
expect(mockSQS.sendMessage).not.toHaveBeenCalledWith(sqsMessage);
123+
expect(SQS).not.toHaveBeenCalled();
110124
});
111125

112126
it('Catch the exception when even copy queue throws exception', async () => {

‎lambdas/functions/webhook/src/sqs/index.ts

+16-12
Original file line numberDiff line numberDiff line change
@@ -50,17 +50,21 @@ export const sendActionRequest = async (message: ActionRequestMessage): Promise<
5050
};
5151

5252
export async function sendWebhookEventToWorkflowJobQueue(message: GithubWorkflowEvent, config: Config): Promise<void> {
53-
if (config.workflowJobEventSecondaryQueue != undefined) {
54-
const sqs = new SQS({ region: process.env.AWS_REGION });
55-
const sqsMessage: SendMessageCommandInput = {
56-
QueueUrl: String(config.workflowJobEventSecondaryQueue),
57-
MessageBody: JSON.stringify(message),
58-
};
59-
logger.debug(`Sending Webhook events to the workflow job queue: ${config.workflowJobEventSecondaryQueue}`);
60-
try {
61-
await sqs.sendMessage(sqsMessage);
62-
} catch (e) {
63-
logger.warn(`Error in sending webhook events to workflow job queue: ${(e as Error).message}`);
64-
}
53+
if (!config.workflowJobEventSecondaryQueue) {
54+
return;
55+
}
56+
57+
const sqs = new SQS({ region: process.env.AWS_REGION });
58+
const sqsMessage: SendMessageCommandInput = {
59+
QueueUrl: String(config.workflowJobEventSecondaryQueue),
60+
MessageBody: JSON.stringify(message),
61+
};
62+
63+
logger.debug(`Sending Webhook events to the workflow job queue: ${config.workflowJobEventSecondaryQueue}`);
64+
65+
try {
66+
await sqs.sendMessage(sqsMessage);
67+
} catch (e) {
68+
logger.warn(`Error in sending webhook events to workflow job queue: ${(e as Error).message}`);
6569
}
6670
}

‎lambdas/libs/aws-ssm-util/package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@
3636
"typescript": "^5.4.5"
3737
},
3838
"dependencies": {
39-
"@aws-sdk/client-ssm": "^3.575.0",
39+
"@aws-sdk/client-ssm": "^3.603.0",
4040
"@aws-sdk/types": "^3.433.0",
4141
"@terraform-aws-github-runner/aws-powertools-util": "*"
4242
},

‎lambdas/libs/aws-ssm-util/src/index.ts

+8-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { GetParameterCommand, PutParameterCommand, SSMClient } from '@aws-sdk/client-ssm';
1+
import { GetParameterCommand, PutParameterCommand, SSMClient, Tag } from '@aws-sdk/client-ssm';
22
import { getTracedAWSV3Client } from '@terraform-aws-github-runner/aws-powertools-util';
33

44
export async function getParameter(parameter_name: string): Promise<string> {
@@ -7,13 +7,19 @@ export async function getParameter(parameter_name: string): Promise<string> {
77
?.Value as string;
88
}
99

10-
export async function putParameter(parameter_name: string, parameter_value: string, secure: boolean): Promise<void> {
10+
export async function putParameter(
11+
parameter_name: string,
12+
parameter_value: string,
13+
secure: boolean,
14+
options: { tags?: Tag[] } = {},
15+
): Promise<void> {
1116
const client = getTracedAWSV3Client(new SSMClient({ region: process.env.AWS_REGION }));
1217
await client.send(
1318
new PutParameterCommand({
1419
Name: parameter_name,
1520
Value: parameter_value,
1621
Type: secure ? 'SecureString' : 'String',
22+
Tags: options.tags,
1723
}),
1824
);
1925
}

0 commit comments

Comments
 (0)