Skip to content
This repository was archived by the owner on Jan 16, 2025. It is now read-only.

Commit a8b33b5

Browse files
npalmnavdeepg2021semantic-release-bot
committed
feat: Added publishing to workflow_job event queue for multi runner module. (#2570)
* feat: Remove support check_run (#2521) * chore: Remove support check_run * format, lint * feat: Remove old scale down mechanism (< 0.19.0) (#2519) fix: Remove old cleanup mechanism (< 0.19.0) * chore: Enable pre releases for next branch * feat: Add multi-runner capability (#2472) * feat: Remove support check_run (#2521) * chore: Remove support check_run * format, lint * feat: Remove old scale down mechanism (< 0.19.0) (#2519) fix: Remove old cleanup mechanism (< 0.19.0) * feat: added changes for multi runner. * fix: region. * fix: more fixes. * tuple to list. * fixes. * fixes. * fixes. * fixes. * fixes. * fixes. * fix: formatting. * fix: formatting. * fix: formatting. * fix: moved some blocks outside runner config. * fix: few more updates * fix: liniting. * fix: updated example output * changed runner group name. * fix: updated the tests. * fix: addressed review comments. * fix: linting issues. * fix: formatting. * fix: updated tf version. * fix: Remove removed prerelease option * Add ubuntu runner to example * refactor: use each instead of count * fix: few small issues. * refactor: syncer to count for multi runner * fix: comments. * fix: added Readme. * fix: errors. * move variable to runner config * fix: updated the readme. * Add todos * feat: added windows runner configuration, completed todos and added the weight for runner config matchers. * chore: Update docs * fix: reverted tf versions. * fix: addressed comments. * fix: missed. * fix: formatting. * Update terraform versions in CI * Update terraform versions in CI * Update docs * fix: coverage. * Update docs * improve test coverage webhook * Apply suggestions from code review * fix: formatting. * fix: fixed merge issues. * fix: syntax. Co-authored-by: Niek Palm <[email protected]> Co-authored-by: Niek Palm <[email protected]> Co-authored-by: navdeepg2021 <[email protected]> * chore(ci): Enable ci for next branch BREAKING CHANGE: - Add option to deploy multiple runner at once, see for details: #2472 #2517 - Drop support check_run event, see for details: #2521 #2517 - Remove pre 0.19.0 scale down mechanism, see details: #2519 #2517 * chore(release): 2.0.0-next.1 [skip ci] * **ci:** * Add multi-runner capability ([#2472](https://github.com/philips-labs/terraform-aws-github-runner/issues/2472)) ([c08b335](c08b335)), closes [#2521](https://github.com/philips-labs/terraform-aws-github-runner/issues/2521) [#2519](https://github.com/philips-labs/terraform-aws-github-runner/issues/2519) * Experimental feature - Duplicate workflow job event to extra queue ([#2268](https://github.com/philips-labs/terraform-aws-github-runner/issues/2268)) ([ac046b8](ac046b8)) * Remove old scale down mechanism (< 0.19.0) ([#2519](https://github.com/philips-labs/terraform-aws-github-runner/issues/2519)) ([721d7c3](721d7c3)) * Remove support check_run ([#2521](https://github.com/philips-labs/terraform-aws-github-runner/issues/2521)) ([272a293](272a293)) * **ci:** Enable ci for next branch ([48769ca](48769ca)), closes [#2472](https://github.com/philips-labs/terraform-aws-github-runner/issues/2472) [#2517](https://github.com/philips-labs/terraform-aws-github-runner/issues/2517) [#2521](https://github.com/philips-labs/terraform-aws-github-runner/issues/2521) [#2517](https://github.com/philips-labs/terraform-aws-github-runner/issues/2517) [#2519](https://github.com/philips-labs/terraform-aws-github-runner/issues/2519) [#2517](https://github.com/philips-labs/terraform-aws-github-runner/issues/2517) * chore: fix release workflow * fix: added changes for publishing events to secondary queue. Co-authored-by: Niek Palm <[email protected]> Co-authored-by: Niek Palm <[email protected]> Co-authored-by: navdeepg2021 <[email protected]> Co-authored-by: semantic-release-bot <[email protected]> feat: added publishing to workflow_job event queue for multi runner module. (#2570) * feat: Remove support check_run (#2521) * chore: Remove support check_run * format, lint * feat: Remove old scale down mechanism (< 0.19.0) (#2519) fix: Remove old cleanup mechanism (< 0.19.0) * chore: Enable pre releases for next branch * feat: Add multi-runner capability (#2472) * feat: Remove support check_run (#2521) * chore: Remove support check_run * format, lint * feat: Remove old scale down mechanism (< 0.19.0) (#2519) fix: Remove old cleanup mechanism (< 0.19.0) * feat: added changes for multi runner. * fix: region. * fix: more fixes. * tuple to list. * fixes. * fixes. * fixes. * fixes. * fixes. * fixes. * fix: formatting. * fix: formatting. * fix: formatting. * fix: moved some blocks outside runner config. * fix: few more updates * fix: liniting. * fix: updated example output * changed runner group name. * fix: updated the tests. * fix: addressed review comments. * fix: linting issues. * fix: formatting. * fix: updated tf version. * fix: Remove removed prerelease option * Add ubuntu runner to example * refactor: use each instead of count * fix: few small issues. * refactor: syncer to count for multi runner * fix: comments. * fix: added Readme. * fix: errors. * move variable to runner config * fix: updated the readme. * Add todos * feat: added windows runner configuration, completed todos and added the weight for runner config matchers. * chore: Update docs * fix: reverted tf versions. * fix: addressed comments. * fix: missed. * fix: formatting. * Update terraform versions in CI * Update terraform versions in CI * Update docs * fix: coverage. * Update docs * improve test coverage webhook * Apply suggestions from code review * fix: formatting. * fix: fixed merge issues. * fix: syntax. Co-authored-by: Niek Palm <[email protected]> Co-authored-by: Niek Palm <[email protected]> Co-authored-by: navdeepg2021 <[email protected]> * chore(ci): Enable ci for next branch BREAKING CHANGE: - Add option to deploy multiple runner at once, see for details: #2472 #2517 - Drop support check_run event, see for details: #2521 #2517 - Remove pre 0.19.0 scale down mechanism, see details: #2519 #2517 * chore(release): 2.0.0-next.1 [skip ci] * **ci:** * Add multi-runner capability ([#2472](https://github.com/philips-labs/terraform-aws-github-runner/issues/2472)) ([c08b335](c08b335)), closes [#2521](https://github.com/philips-labs/terraform-aws-github-runner/issues/2521) [#2519](https://github.com/philips-labs/terraform-aws-github-runner/issues/2519) * Experimental feature - Duplicate workflow job event to extra queue ([#2268](https://github.com/philips-labs/terraform-aws-github-runner/issues/2268)) ([ac046b8](ac046b8)) * Remove old scale down mechanism (< 0.19.0) ([#2519](https://github.com/philips-labs/terraform-aws-github-runner/issues/2519)) ([721d7c3](721d7c3)) * Remove support check_run ([#2521](https://github.com/philips-labs/terraform-aws-github-runner/issues/2521)) ([272a293](272a293)) * **ci:** Enable ci for next branch ([48769ca](48769ca)), closes [#2472](https://github.com/philips-labs/terraform-aws-github-runner/issues/2472) [#2517](https://github.com/philips-labs/terraform-aws-github-runner/issues/2517) [#2521](https://github.com/philips-labs/terraform-aws-github-runner/issues/2521) [#2517](https://github.com/philips-labs/terraform-aws-github-runner/issues/2517) [#2519](https://github.com/philips-labs/terraform-aws-github-runner/issues/2519) [#2517](https://github.com/philips-labs/terraform-aws-github-runner/issues/2517) * chore: fix release workflow * fix: added changes for publishing events to secondary queue. Co-authored-by: Niek Palm <[email protected]> Co-authored-by: Niek Palm <[email protected]> Co-authored-by: navdeepg2021 <[email protected]> Co-authored-by: semantic-release-bot <[email protected]>
1 parent 8c04666 commit a8b33b5

File tree

6 files changed

+49
-2
lines changed

6 files changed

+49
-2
lines changed

Diff for: examples/multi-runner/main.tf

+2
Original file line numberDiff line numberDiff line change
@@ -159,7 +159,9 @@ module "multi-runner" {
159159
# runner_binaries_syncer_lambda_zip = "lambdas-download/runner-binaries-syncer.zip"
160160
# runners_lambda_zip = "lambdas-download/runners.zip"
161161

162+
# enable_workflow_job_events_queue = true
162163
# override delay of events in seconds
163164

164165
# log_level = "debug"
166+
165167
}

Diff for: modules/multi-runner/README.md

+3
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,7 @@ No requirements.
103103
|------|------|
104104
| [aws_sqs_queue.queued_builds](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue) | resource |
105105
| [aws_sqs_queue.queued_builds_dlq](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue) | resource |
106+
| [aws_sqs_queue.webhook_events_workflow_job_queue](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue) | resource |
106107
| [aws_sqs_queue_policy.build_queue_dlq_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue_policy) | resource |
107108
| [aws_sqs_queue_policy.build_queue_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue_policy) | resource |
108109
| [random_string.random](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) | resource |
@@ -116,6 +117,7 @@ No requirements.
116117
| <a name="input_aws_region"></a> [aws\_region](#input\_aws\_region) | AWS region. | `string` | n/a | yes |
117118
| <a name="input_cloudwatch_config"></a> [cloudwatch\_config](#input\_cloudwatch\_config) | (optional) Replaces the module default cloudwatch log config. See https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Agent-Configuration-File-Details.html for details. | `string` | `null` | no |
118119
| <a name="input_enable_managed_runner_security_group"></a> [enable\_managed\_runner\_security\_group](#input\_enable\_managed\_runner\_security\_group) | Enabling the default managed security group creation. Unmanaged security groups can be specified via `runner_additional_security_group_ids`. | `bool` | `true` | no |
120+
| <a name="input_enable_workflow_job_events_queue"></a> [enable\_workflow\_job\_events\_queue](#input\_enable\_workflow\_job\_events\_queue) | Enabling this experimental feature will create a secondory sqs queue to wich a copy of the workflow\_job event will be delivered. | `bool` | `false` | no |
119121
| <a name="input_ghes_ssl_verify"></a> [ghes\_ssl\_verify](#input\_ghes\_ssl\_verify) | GitHub Enterprise SSL verification. Set to 'false' when custom certificate (chains) is used for GitHub Enterprise Server (insecure). | `bool` | `true` | no |
120122
| <a name="input_ghes_url"></a> [ghes\_url](#input\_ghes\_url) | GitHub Enterprise Server URL. Example: https://github.internal.co - DO NOT SET IF USING PUBLIC GITHUB | `string` | `null` | no |
121123
| <a name="input_github_app"></a> [github\_app](#input\_github\_app) | GitHub app parameters, see your github app. Ensure the key is the base64-encoded `.pem` file (the output of `base64 app.private-key.pem`, not the content of `private-key.pem`). | <pre>object({<br> key_base64 = string<br> id = string<br> webhook_secret = string<br> })</pre> | n/a | yes |
@@ -161,6 +163,7 @@ No requirements.
161163
| <a name="input_webhook_lambda_s3_object_version"></a> [webhook\_lambda\_s3\_object\_version](#input\_webhook\_lambda\_s3\_object\_version) | S3 object version for webhook lambda function. Useful if S3 versioning is enabled on source bucket. | `any` | `null` | no |
162164
| <a name="input_webhook_lambda_timeout"></a> [webhook\_lambda\_timeout](#input\_webhook\_lambda\_timeout) | Time out of the lambda in seconds. | `number` | `10` | no |
163165
| <a name="input_webhook_lambda_zip"></a> [webhook\_lambda\_zip](#input\_webhook\_lambda\_zip) | File location of the webhook lambda zip file. | `string` | `null` | no |
166+
| <a name="input_workflow_job_queue_configuration"></a> [workflow\_job\_queue\_configuration](#input\_workflow\_job\_queue\_configuration) | Configuration options for workflow job queue which is only applicable if the flag enable\_workflow\_job\_events\_queue is set to true. | <pre>object({<br> delay_seconds = number<br> visibility_timeout_seconds = number<br> message_retention_seconds = number<br> })</pre> | <pre>{<br> "delay_seconds": null,<br> "message_retention_seconds": null,<br> "visibility_timeout_seconds": null<br>}</pre> | no |
164167

165168
## Outputs
166169

Diff for: modules/multi-runner/queues.tf

+19
Original file line numberDiff line numberDiff line change
@@ -70,3 +70,22 @@ resource "aws_sqs_queue_policy" "build_queue_dlq_policy" {
7070
queue_url = aws_sqs_queue.queued_builds_dlq[each.key].id
7171
policy = data.aws_iam_policy_document.deny_unsecure_transport.json
7272
}
73+
74+
resource "aws_sqs_queue" "webhook_events_workflow_job_queue" {
75+
count = var.enable_workflow_job_events_queue ? 1 : 0
76+
name = "${var.prefix}-webhook_events_workflow_job_queue"
77+
delay_seconds = var.workflow_job_queue_configuration.delay_seconds
78+
visibility_timeout_seconds = var.workflow_job_queue_configuration.visibility_timeout_seconds
79+
message_retention_seconds = var.workflow_job_queue_configuration.message_retention_seconds
80+
fifo_queue = false
81+
receive_wait_time_seconds = 0
82+
content_based_deduplication = false
83+
redrive_policy = null
84+
85+
sqs_managed_sse_enabled = var.queue_encryption.sqs_managed_sse_enabled
86+
kms_master_key_id = var.queue_encryption.kms_master_key_id
87+
kms_data_key_reuse_period_seconds = var.queue_encryption.kms_data_key_reuse_period_seconds
88+
89+
tags = var.tags
90+
}
91+

Diff for: modules/multi-runner/variables.tf

+21
Original file line numberDiff line numberDiff line change
@@ -489,3 +489,24 @@ variable "pool_lambda_reserved_concurrent_executions" {
489489
type = number
490490
default = 1
491491
}
492+
493+
variable "enable_workflow_job_events_queue" {
494+
description = "Enabling this experimental feature will create a secondory sqs queue to wich a copy of the workflow_job event will be delivered."
495+
type = bool
496+
default = false
497+
}
498+
499+
variable "workflow_job_queue_configuration" {
500+
description = "Configuration options for workflow job queue which is only applicable if the flag enable_workflow_job_events_queue is set to true."
501+
type = object({
502+
delay_seconds = number
503+
visibility_timeout_seconds = number
504+
message_retention_seconds = number
505+
})
506+
default = {
507+
"delay_seconds" : null,
508+
"visibility_timeout_seconds" : null,
509+
"message_retention_seconds" : null
510+
}
511+
}
512+

Diff for: modules/multi-runner/webhook.tf

+2
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ module "webhook" {
66

77
runner_config = local.runner_config
88
github_app_webhook_secret_arn = module.ssm.parameters.github_app_webhook_secret.arn
9+
sqs_workflow_job_queue = length(aws_sqs_queue.webhook_events_workflow_job_queue) > 0 ? aws_sqs_queue.webhook_events_workflow_job_queue[0] : null
10+
911

1012
lambda_s3_bucket = var.lambda_s3_bucket
1113
webhook_lambda_s3_key = var.webhook_lambda_s3_key

Diff for: modules/webhook/lambdas/webhook/src/webhook/handler.ts

+2-2
Original file line numberDiff line numberDiff line change
@@ -63,10 +63,10 @@ export async function handle(headers: IncomingHttpHeaders, body: string): Promis
6363

6464
const workflowJobEvent = payload as WorkflowJobEvent;
6565
response = await handleWorkflowJob(workflowJobEvent, githubEvent, queuesConfig);
66-
await sendWorkflowJobEvents(githubEvent, workflowJobEvent);
66+
await sendWorkflowJobEvents(workflowJobEvent);
6767
return response;
6868
}
69-
async function sendWorkflowJobEvents(githubEvent: string, workflowEventPayload: WorkflowJobEvent) {
69+
async function sendWorkflowJobEvents(workflowEventPayload: WorkflowJobEvent) {
7070
await sendWebhookEventToWorkflowJobQueue({
7171
workflowJobEvent: workflowEventPayload,
7272
});

0 commit comments

Comments
 (0)