diff --git a/.tflint.hcl b/.tflint.hcl index bcebefe93e..ccec5fb4a9 100644 --- a/.tflint.hcl +++ b/.tflint.hcl @@ -1,6 +1,6 @@ config { format = "compact" - module = true + call_module_type = "local" } plugin "aws" { diff --git a/README.md b/README.md index 2d734da30d..2655b817d6 100644 --- a/README.md +++ b/README.md @@ -110,10 +110,8 @@ Talk to the forestkeepers in the `runners-channel` on Slack. |------|------| | [aws_sqs_queue.queued_builds](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue) | resource | | [aws_sqs_queue.queued_builds_dlq](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue) | resource | -| [aws_sqs_queue.webhook_events_workflow_job_queue](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue) | resource | | [aws_sqs_queue_policy.build_queue_dlq_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue_policy) | resource | | [aws_sqs_queue_policy.build_queue_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue_policy) | resource | -| [aws_sqs_queue_policy.webhook_events_workflow_job_queue_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue_policy) | resource | | [random_string.random](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) | resource | | [aws_iam_policy_document.deny_unsecure_transport](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | @@ -156,7 +154,6 @@ Talk to the forestkeepers in the `runners-channel` on Slack. | [enable\_ssm\_on\_runners](#input\_enable\_ssm\_on\_runners) | Enable to allow access to the runner instances for debugging purposes via SSM. Note that this adds additional permissions to the runner instances. | `bool` | `false` | no | | [enable\_user\_data\_debug\_logging\_runner](#input\_enable\_user\_data\_debug\_logging\_runner) | Option to enable debug logging for user-data, this logs all secrets as well. | `bool` | `false` | no | | [enable\_userdata](#input\_enable\_userdata) | Should the userdata script be enabled for the runner. Set this to false if you are using your own prebuilt AMI. | `bool` | `true` | no | -| [enable\_workflow\_job\_events\_queue](#input\_enable\_workflow\_job\_events\_queue) | Enabling this experimental feature will create a secondary SQS queue to which a copy of the workflow\_job event will be delivered. | `bool` | `false` | no | | [eventbridge](#input\_eventbridge) | Enable the use of EventBridge by the module. By enabling this feature events will be put on the EventBridge by the webhook instead of directly dispatching to queues for scaling.

`enable`: Enable the EventBridge feature.
`accept_events`: List can be used to only allow specific events to be putted on the EventBridge. By default all events, empty list will be be interpreted as all events. | 
object({
    enable        = optional(bool, false)
    accept_events = optional(list(string), null)
  })
| `{}` | no | | [ghes\_ssl\_verify](#input\_ghes\_ssl\_verify) | GitHub Enterprise SSL verification. Set to 'false' when custom certificate (chains) is used for GitHub Enterprise Server (insecure). | `bool` | `true` | no | | [ghes\_url](#input\_ghes\_url) | GitHub Enterprise Server URL. Example: https://github.internal.co - DO NOT SET IF USING PUBLIC GITHUB | `string` | `null` | no | @@ -251,7 +248,6 @@ Talk to the forestkeepers in the `runners-channel` on Slack. | [webhook\_lambda\_s3\_object\_version](#input\_webhook\_lambda\_s3\_object\_version) | S3 object version for webhook lambda function. Useful if S3 versioning is enabled on source bucket. | `string` | `null` | no | | [webhook\_lambda\_timeout](#input\_webhook\_lambda\_timeout) | Time out of the webhook lambda in seconds. | `number` | `10` | no | | [webhook\_lambda\_zip](#input\_webhook\_lambda\_zip) | File location of the webhook lambda zip file. | `string` | `null` | no | -| [workflow\_job\_queue\_configuration](#input\_workflow\_job\_queue\_configuration) | Configuration options for workflow job queue which is only applicable if the flag enable\_workflow\_job\_events\_queue is set to true. | 
object({
    delay_seconds              = number
    visibility_timeout_seconds = number
    message_retention_seconds  = number
  })
| 
{
  "delay_seconds": null,
  "message_retention_seconds": null,
  "visibility_timeout_seconds": null
}
| no | ## Outputs diff --git a/docs/configuration.md b/docs/configuration.md index 6d74b1fe6a..3c3ce680b2 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -332,23 +332,59 @@ resource "aws_iam_role_policy" "event_rule_firehose_role" { ### Queue to publish workflow job events -!!! warning "Deprecated +!!! warning "Removed - This fearure will be removed since we introducing the EventBridge. Same functinallity can be implemented by adding a rule to the EventBridge to forward `workflow_job` events to the SQS queue. + This feaTure will be removed since we introducing the EventBridge. Same functionality can be implemented by adding a rule to the EventBridge to forward `workflow_job` events to the SQS queue. -This queue is an experimental feature to allow you to receive a copy of the wokflow_jobs events sent by the GitHub App. This can be used to calculate a matrix or monitor the system. +Below an example how you can sent all `workflow_job` with action `in_progress` to a SQS queue. -To enable the feature set `enable_workflow_job_events_queue = true`. Be aware though, this feature is experimental! +```hcl -Messages received on the queue are using the same format as published by GitHub wrapped in a property `workflowJobEvent`. +resource "aws_cloudwatch_event_rule" "workflow_job_in_progress" { + name = "workflow-job-in-progress" + event_bus_name = modules.runners.webhook.eventbridge.name # The name of the event bus output by the module -``` -export interface GithubWorkflowEvent { - workflowJobEvent: WorkflowJobEvent; + event_pattern = < { describe('ConfigWebhook', () => { it('should load config successfully', async () => { process.env.REPOSITORY_ALLOW_LIST = '["repo1", "repo2"]'; - process.env.SQS_WORKFLOW_JOB_QUEUE = 'secondary-queue'; - process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = '/path/to/matcher/config'; process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET = '/path/to/webhook/secret'; + process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = '/path/to/matcher/config'; const matcherConfig = [ { id: '1', @@ -121,7 +120,6 @@ describe('ConfigLoader Tests', () => { const config: ConfigWebhook = await ConfigWebhook.load(); expect(config.repositoryAllowList).toEqual(['repo1', 'repo2']); - expect(config.workflowJobEventSecondaryQueue).toBe('secondary-queue'); expect(config.matcherConfig).toEqual(matcherConfig); expect(config.webhookSecret).toBe('secret'); }); diff --git a/lambdas/functions/webhook/src/ConfigLoader.ts b/lambdas/functions/webhook/src/ConfigLoader.ts index 7dc3a3b695..646acfaa53 100644 --- a/lambdas/functions/webhook/src/ConfigLoader.ts +++ b/lambdas/functions/webhook/src/ConfigLoader.ts @@ -95,7 +95,6 @@ export class ConfigWebhook extends BaseConfig { async loadConfig(): Promise { this.loadEnvVar(process.env.REPOSITORY_ALLOW_LIST, 'repositoryAllowList', []); - this.loadEnvVar(process.env.SQS_WORKFLOW_JOB_QUEUE, 'workflowJobEventSecondaryQueue', ''); await Promise.all([ this.loadParameter(process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH, 'matcherConfig'), @@ -129,7 +128,6 @@ export class ConfigDispatcher extends BaseConfig { async loadConfig(): Promise { this.loadEnvVar(process.env.REPOSITORY_ALLOW_LIST, 'repositoryAllowList', []); - this.loadEnvVar(process.env.SQS_WORKFLOW_JOB_QUEUE, 'workflowJobEventSecondaryQueue', ''); await this.loadParameter(process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH, 'matcherConfig'); validateRunnerMatcherConfig(this); diff --git a/lambdas/functions/webhook/src/modules.d.ts b/lambdas/functions/webhook/src/modules.d.ts index a3bc22c3e0..76a72660c0 100644 --- a/lambdas/functions/webhook/src/modules.d.ts +++ b/lambdas/functions/webhook/src/modules.d.ts @@ -7,6 +7,5 @@ declare namespace NodeJS { REPOSITORY_ALLOW_LIST: string; RUNNER_LABELS: string; ACCEPT_EVENTS: string; - SQS_WORKFLOW_JOB_QUEUE: string; } } diff --git a/lambdas/functions/webhook/src/runners/dispatch.test.ts b/lambdas/functions/webhook/src/runners/dispatch.test.ts index 88d6aa3f55..902022c410 100644 --- a/lambdas/functions/webhook/src/runners/dispatch.test.ts +++ b/lambdas/functions/webhook/src/runners/dispatch.test.ts @@ -14,7 +14,6 @@ import { logger } from '@aws-github-runner/aws-powertools-util'; jest.mock('../sqs'); jest.mock('@aws-github-runner/aws-ssm-util'); -const sendWebhookEventToWorkflowJobQueueMock = jest.mocked(sendActionRequest); const GITHUB_APP_WEBHOOK_SECRET = 'TEST_SECRET'; const cleanEnv = process.env; @@ -56,7 +55,6 @@ describe('Dispatcher', () => { statusCode: 403, }); expect(sendActionRequest).not.toHaveBeenCalled(); - expect(sendWebhookEventToWorkflowJobQueueMock).not.toHaveBeenCalled(); }); it('should handle workflow_job events without installation id', async () => { @@ -65,7 +63,6 @@ describe('Dispatcher', () => { const resp = await dispatch(event, 'workflow_job', config); expect(resp.statusCode).toBe(201); expect(sendActionRequest).toHaveBeenCalled(); - expect(sendWebhookEventToWorkflowJobQueueMock).toHaveBeenCalled(); }); it('should handle workflow_job events from allow listed repositories', async () => { @@ -74,7 +71,6 @@ describe('Dispatcher', () => { const resp = await dispatch(event, 'workflow_job', config); expect(resp.statusCode).toBe(201); expect(sendActionRequest).toHaveBeenCalled(); - expect(sendWebhookEventToWorkflowJobQueueMock).toHaveBeenCalled(); }); it('should match labels', async () => { @@ -108,7 +104,6 @@ describe('Dispatcher', () => { queueFifo: false, repoOwnerType: 'Organization', }); - expect(sendWebhookEventToWorkflowJobQueueMock).toHaveBeenCalled(); }); it('should sort matcher with exact first.', async () => { @@ -157,7 +152,6 @@ describe('Dispatcher', () => { queueFifo: false, repoOwnerType: 'Organization', }); - expect(sendWebhookEventToWorkflowJobQueueMock).toHaveBeenCalled(); }); it('should not accept jobs where not all labels are supported (single matcher).', async () => { @@ -181,7 +175,6 @@ describe('Dispatcher', () => { const resp = await dispatch(event, 'workflow_job', config); expect(resp.statusCode).toBe(202); expect(sendActionRequest).not.toHaveBeenCalled(); - expect(sendWebhookEventToWorkflowJobQueueMock).not.toHaveBeenCalled(); }); }); diff --git a/lambdas/functions/webhook/src/runners/dispatch.ts b/lambdas/functions/webhook/src/runners/dispatch.ts index 91f9126413..d628f4c846 100644 --- a/lambdas/functions/webhook/src/runners/dispatch.ts +++ b/lambdas/functions/webhook/src/runners/dispatch.ts @@ -2,7 +2,7 @@ import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; import { WorkflowJobEvent } from '@octokit/webhooks-types'; import { Response } from '../lambda'; -import { RunnerMatcherConfig, sendActionRequest, sendWebhookEventToWorkflowJobQueue } from '../sqs'; +import { RunnerMatcherConfig, sendActionRequest } from '../sqs'; import ValidationError from '../ValidationError'; import { ConfigDispatcher, ConfigWebhook } from '../ConfigLoader'; @@ -15,10 +15,7 @@ export async function dispatch( ): Promise { validateRepoInAllowList(event, config); - const result = await handleWorkflowJob(event, eventType, config.matcherConfig!); - await sendWebhookEventToWorkflowJobQueue({ workflowJobEvent: event }, config); - - return result; + return await handleWorkflowJob(event, eventType, config.matcherConfig!); } function validateRepoInAllowList(event: WorkflowJobEvent, config: ConfigDispatcher) { diff --git a/lambdas/functions/webhook/src/sqs/index.test.ts b/lambdas/functions/webhook/src/sqs/index.test.ts index f8fe64f2e4..be539ef0c3 100644 --- a/lambdas/functions/webhook/src/sqs/index.test.ts +++ b/lambdas/functions/webhook/src/sqs/index.test.ts @@ -1,9 +1,5 @@ import { SendMessageCommandInput } from '@aws-sdk/client-sqs'; - -import { ActionRequestMessage, GithubWorkflowEvent, sendActionRequest, sendWebhookEventToWorkflowJobQueue } from '.'; -import workflowjob_event from '../../test/resources/github_workflowjob_event.json'; -import { getParameter } from '@aws-github-runner/aws-ssm-util'; -import { mocked } from 'jest-mock'; +import { ActionRequestMessage, sendActionRequest } from '.'; const mockSQS = { sendMessage: jest.fn(() => { @@ -15,9 +11,6 @@ jest.mock('@aws-sdk/client-sqs', () => ({ })); jest.mock('@aws-github-runner/aws-ssm-util'); -import { SQS } from '@aws-sdk/client-sqs'; -import { ConfigDispatcher, ConfigWebhook } from '../ConfigLoader'; - describe('Test sending message to SQS.', () => { const queueUrl = 'https://sqs.eu-west-1.amazonaws.com/123456789/queued-builds'; const message = { @@ -72,62 +65,3 @@ describe('Test sending message to SQS.', () => { await expect(result).resolves.not.toThrow(); }); }); - -describe('Test sending message to SQS.', () => { - const message: GithubWorkflowEvent = { - workflowJobEvent: JSON.parse(JSON.stringify(workflowjob_event)), - }; - const sqsMessage: SendMessageCommandInput = { - QueueUrl: 'https://sqs.eu-west-1.amazonaws.com/123456789/webhook_events_workflow_job_queue', - MessageBody: JSON.stringify(message), - }; - beforeEach(() => { - ConfigDispatcher.reset(); - const mockedGet = mocked(getParameter); - mockedGet.mockResolvedValue('["abc"]'); - }); - afterEach(() => { - jest.clearAllMocks(); - }); - - it('sends webhook events to workflow job queue', async () => { - // Arrange - process.env.SQS_WORKFLOW_JOB_QUEUE = sqsMessage.QueueUrl || ''; - const config: ConfigWebhook = await ConfigWebhook.load(); - - // Act - const result = sendWebhookEventToWorkflowJobQueue(message, config); - - // Assert - expect(mockSQS.sendMessage).toHaveBeenCalledWith(sqsMessage); - await expect(result).resolves.not.toThrow(); - }); - - it('Does not send webhook events to workflow job event copy queue when job queue is not in environment', async () => { - // Arrange - process.env.SQS_WORKFLOW_JOB_QUEUE = ''; - const config: ConfigDispatcher = await ConfigDispatcher.load(); - - // Act - await sendWebhookEventToWorkflowJobQueue(message, config); - - // Assert - expect(SQS).not.toHaveBeenCalled(); - }); - - it('Catch the exception when even copy queue throws exception', async () => { - // Arrange - process.env.SQS_WORKFLOW_JOB_QUEUE = sqsMessage.QueueUrl || ''; - const config: ConfigDispatcher = await ConfigDispatcher.load(); - - const mockSQS = { - sendMessage: jest.fn(() => { - throw new Error(); - }), - }; - jest.mock('aws-sdk', () => ({ - SQS: jest.fn().mockImplementation(() => mockSQS), - })); - await expect(sendWebhookEventToWorkflowJobQueue(message, config)).resolves.not.toThrow(); - }); -}); diff --git a/lambdas/functions/webhook/src/sqs/index.ts b/lambdas/functions/webhook/src/sqs/index.ts index 14f61f40e3..2d9b3ed2dd 100644 --- a/lambdas/functions/webhook/src/sqs/index.ts +++ b/lambdas/functions/webhook/src/sqs/index.ts @@ -1,7 +1,6 @@ import { SQS, SendMessageCommandInput } from '@aws-sdk/client-sqs'; import { WorkflowJobEvent } from '@octokit/webhooks-types'; import { createChildLogger, getTracedAWSV3Client } from '@aws-github-runner/aws-powertools-util'; -import { ConfigDispatcher } from '../ConfigLoader'; const logger = createChildLogger('sqs'); @@ -49,26 +48,3 @@ export const sendActionRequest = async (message: ActionRequestMessage): Promise< await sqs.sendMessage(sqsMessage); }; - -export async function sendWebhookEventToWorkflowJobQueue( - message: GithubWorkflowEvent, - config: ConfigDispatcher, -): Promise { - if (!config.workflowJobEventSecondaryQueue) { - return; - } - - const sqs = new SQS({ region: process.env.AWS_REGION }); - const sqsMessage: SendMessageCommandInput = { - QueueUrl: String(config.workflowJobEventSecondaryQueue), - MessageBody: JSON.stringify(message), - }; - - logger.info(`Sending event to the workflow job queue: ${config.workflowJobEventSecondaryQueue}`); - - try { - await sqs.sendMessage(sqsMessage); - } catch (e) { - logger.warn(`Error in sending webhook events to workflow job queue: ${(e as Error).message}`); - } -} diff --git a/main.tf b/main.tf index 12f0bd96bb..3523e9ff3a 100644 --- a/main.tf +++ b/main.tf @@ -51,12 +51,6 @@ resource "aws_sqs_queue_policy" "build_queue_policy" { policy = data.aws_iam_policy_document.deny_unsecure_transport.json } -resource "aws_sqs_queue_policy" "webhook_events_workflow_job_queue_policy" { - count = var.enable_workflow_job_events_queue ? 1 : 0 - queue_url = aws_sqs_queue.webhook_events_workflow_job_queue[0].id - policy = data.aws_iam_policy_document.deny_unsecure_transport.json -} - resource "aws_sqs_queue" "queued_builds" { name = "${var.prefix}-queued-builds${var.enable_fifo_build_queue ? ".fifo" : ""}" delay_seconds = var.delay_webhook_event @@ -77,24 +71,6 @@ resource "aws_sqs_queue" "queued_builds" { tags = var.tags } -resource "aws_sqs_queue" "webhook_events_workflow_job_queue" { - count = var.enable_workflow_job_events_queue ? 1 : 0 - name = "${var.prefix}-webhook_events_workflow_job_queue" - delay_seconds = var.workflow_job_queue_configuration.delay_seconds - visibility_timeout_seconds = var.workflow_job_queue_configuration.visibility_timeout_seconds - message_retention_seconds = var.workflow_job_queue_configuration.message_retention_seconds - fifo_queue = false - receive_wait_time_seconds = 0 - content_based_deduplication = false - redrive_policy = null - - sqs_managed_sse_enabled = var.queue_encryption.sqs_managed_sse_enabled - kms_master_key_id = var.queue_encryption.kms_master_key_id - kms_data_key_reuse_period_seconds = var.queue_encryption.kms_data_key_reuse_period_seconds - - tags = var.tags -} - resource "aws_sqs_queue_policy" "build_queue_dlq_policy" { count = var.redrive_build_queue.enabled ? 1 : 0 queue_url = aws_sqs_queue.queued_builds.id @@ -145,7 +121,6 @@ module "webhook" { } } matcher_config_parameter_store_tier = var.matcher_config_parameter_store_tier - sqs_workflow_job_queue = length(aws_sqs_queue.webhook_events_workflow_job_queue) > 0 ? aws_sqs_queue.webhook_events_workflow_job_queue[0] : null github_app_parameters = { webhook_secret = module.ssm.parameters.github_app_webhook_secret diff --git a/modules/multi-runner/README.md b/modules/multi-runner/README.md index d47ea60c6f..a7557155f9 100644 --- a/modules/multi-runner/README.md +++ b/modules/multi-runner/README.md @@ -106,10 +106,8 @@ module "multi-runner" { |------|------| | [aws_sqs_queue.queued_builds](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue) | resource | | [aws_sqs_queue.queued_builds_dlq](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue) | resource | -| [aws_sqs_queue.webhook_events_workflow_job_queue](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue) | resource | | [aws_sqs_queue_policy.build_queue_dlq_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue_policy) | resource | | [aws_sqs_queue_policy.build_queue_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue_policy) | resource | -| [aws_sqs_queue_policy.webhook_events_workflow_job_queue_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue_policy) | resource | | [random_string.random](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) | resource | | [aws_iam_policy_document.deny_unsecure_transport](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | @@ -131,7 +129,6 @@ module "multi-runner" { | [enable\_ami\_housekeeper](#input\_enable\_ami\_housekeeper) | Option to disable the lambda to clean up old AMIs. | `bool` | `false` | no | | [enable\_managed\_runner\_security\_group](#input\_enable\_managed\_runner\_security\_group) | Enabling the default managed security group creation. Unmanaged security groups can be specified via `runner_additional_security_group_ids`. | `bool` | `true` | no | | [enable\_metrics\_control\_plane](#input\_enable\_metrics\_control\_plane) | (Experimental) Enable or disable the metrics for the module. Feature can change or renamed without a major release. | `bool` | `false` | no | -| [enable\_workflow\_job\_events\_queue](#input\_enable\_workflow\_job\_events\_queue) | Enabling this experimental feature will create a secondary SQS queue to which a copy of the workflow\_job event will be delivered. | `bool` | `false` | no | | [eventbridge](#input\_eventbridge) | Enable the use of EventBridge by the module. By enabling this feature events will be put on the EventBridge by the webhook instead of directly dispatching to queues for scaling. | 
object({
    enable        = optional(bool, false)
    accept_events = optional(list(string), [])
  })
| `{}` | no | | [ghes\_ssl\_verify](#input\_ghes\_ssl\_verify) | GitHub Enterprise SSL verification. Set to 'false' when custom certificate (chains) is used for GitHub Enterprise Server (insecure). | `bool` | `true` | no | | [ghes\_url](#input\_ghes\_url) | GitHub Enterprise Server URL. Example: https://github.internal.co - DO NOT SET IF USING PUBLIC GITHUB | `string` | `null` | no | @@ -190,7 +187,6 @@ module "multi-runner" { | [webhook\_lambda\_s3\_object\_version](#input\_webhook\_lambda\_s3\_object\_version) | S3 object version for webhook lambda function. Useful if S3 versioning is enabled on source bucket. | `string` | `null` | no | | [webhook\_lambda\_timeout](#input\_webhook\_lambda\_timeout) | Time out of the lambda in seconds. | `number` | `10` | no | | [webhook\_lambda\_zip](#input\_webhook\_lambda\_zip) | File location of the webhook lambda zip file. | `string` | `null` | no | -| [workflow\_job\_queue\_configuration](#input\_workflow\_job\_queue\_configuration) | Configuration options for workflow job queue which is only applicable if the flag enable\_workflow\_job\_events\_queue is set to true. | 
object({
    delay_seconds              = number
    visibility_timeout_seconds = number
    message_retention_seconds  = number
  })
| 
{
  "delay_seconds": null,
  "message_retention_seconds": null,
  "visibility_timeout_seconds": null
}
| no | ## Outputs @@ -199,7 +195,6 @@ module "multi-runner" { | [binaries\_syncer\_map](#output\_binaries\_syncer\_map) | n/a | | [instance\_termination\_handler](#output\_instance\_termination\_handler) | n/a | | [instance\_termination\_watcher](#output\_instance\_termination\_watcher) | n/a | -| [queues](#output\_queues) | SQS queues. | | [runners\_map](#output\_runners\_map) | n/a | | [ssm\_parameters](#output\_ssm\_parameters) | n/a | | [webhook](#output\_webhook) | n/a | diff --git a/modules/multi-runner/outputs.tf b/modules/multi-runner/outputs.tf index 0a7b99243f..42758c0652 100644 --- a/modules/multi-runner/outputs.tf +++ b/modules/multi-runner/outputs.tf @@ -48,13 +48,6 @@ output "ssm_parameters" { value = module.ssm.parameters } -output "queues" { - description = "SQS queues." - value = { - webhook_workflow_job_queue = try(aws_sqs_queue.webhook_events_workflow_job_queue[*].arn, "") - } -} - output "instance_termination_watcher" { value = var.instance_termination_watcher.enable && var.instance_termination_watcher.features.enable_spot_termination_notification_watcher ? { lambda = module.instance_termination_watcher[0].spot_termination_notification.lambda diff --git a/modules/multi-runner/queues.tf b/modules/multi-runner/queues.tf index cc53e36630..9def57a316 100644 --- a/modules/multi-runner/queues.tf +++ b/modules/multi-runner/queues.tf @@ -69,28 +69,3 @@ resource "aws_sqs_queue_policy" "build_queue_dlq_policy" { queue_url = aws_sqs_queue.queued_builds_dlq[each.key].id policy = data.aws_iam_policy_document.deny_unsecure_transport.json } - -resource "aws_sqs_queue_policy" "webhook_events_workflow_job_queue_policy" { - count = var.enable_workflow_job_events_queue ? 1 : 0 - queue_url = aws_sqs_queue.webhook_events_workflow_job_queue[0].id - policy = data.aws_iam_policy_document.deny_unsecure_transport.json -} - -resource "aws_sqs_queue" "webhook_events_workflow_job_queue" { - count = var.enable_workflow_job_events_queue ? 1 : 0 - name = "${var.prefix}-webhook_events_workflow_job_queue" - delay_seconds = var.workflow_job_queue_configuration.delay_seconds - visibility_timeout_seconds = var.workflow_job_queue_configuration.visibility_timeout_seconds - message_retention_seconds = var.workflow_job_queue_configuration.message_retention_seconds - fifo_queue = false - receive_wait_time_seconds = 0 - content_based_deduplication = false - redrive_policy = null - - sqs_managed_sse_enabled = var.queue_encryption.sqs_managed_sse_enabled - kms_master_key_id = var.queue_encryption.kms_master_key_id - kms_data_key_reuse_period_seconds = var.queue_encryption.kms_data_key_reuse_period_seconds - - tags = var.tags -} - diff --git a/modules/multi-runner/variables.tf b/modules/multi-runner/variables.tf index a9d0a9f906..ef104ef50c 100644 --- a/modules/multi-runner/variables.tf +++ b/modules/multi-runner/variables.tf @@ -550,26 +550,6 @@ variable "pool_lambda_reserved_concurrent_executions" { default = 1 } -variable "enable_workflow_job_events_queue" { - description = "Enabling this experimental feature will create a secondary SQS queue to which a copy of the workflow_job event will be delivered." - type = bool - default = false -} - -variable "workflow_job_queue_configuration" { - description = "Configuration options for workflow job queue which is only applicable if the flag enable_workflow_job_events_queue is set to true." - type = object({ - delay_seconds = number - visibility_timeout_seconds = number - message_retention_seconds = number - }) - default = { - "delay_seconds" : null, - "visibility_timeout_seconds" : null, - "message_retention_seconds" : null - } -} - variable "ssm_paths" { description = "The root path used in SSM to store configuration and secreets." type = object({ diff --git a/modules/multi-runner/webhook.tf b/modules/multi-runner/webhook.tf index 9e70ca81a2..c819b763e0 100644 --- a/modules/multi-runner/webhook.tf +++ b/modules/multi-runner/webhook.tf @@ -11,7 +11,6 @@ module "webhook" { root = local.ssm_root_path webhook = var.ssm_paths.webhook } - sqs_workflow_job_queue = length(aws_sqs_queue.webhook_events_workflow_job_queue) > 0 ? aws_sqs_queue.webhook_events_workflow_job_queue[0] : null github_app_parameters = { webhook_secret = module.ssm.parameters.github_app_webhook_secret diff --git a/modules/webhook/README.md b/modules/webhook/README.md index 4408bc56af..c066cc51c9 100644 --- a/modules/webhook/README.md +++ b/modules/webhook/README.md @@ -88,7 +88,6 @@ yarn run dist | [role\_path](#input\_role\_path) | The path that will be added to the role; if not set, the environment name will be used. | `string` | `null` | no | | [role\_permissions\_boundary](#input\_role\_permissions\_boundary) | Permissions boundary that will be added to the created role for the lambda. | `string` | `null` | no | | [runner\_matcher\_config](#input\_runner\_matcher\_config) | SQS queue to publish accepted build events based on the runner type. When exact match is disabled the webhook accepts the event if one of the workflow job labels is part of the matcher. The priority defines the order the matchers are applied. | 
map(object({
    arn  = string
    id   = string
    fifo = bool
    matcherConfig = object({
      labelMatchers = list(list(string))
      exactMatch    = bool
      priority      = optional(number, 999)
    })
  }))
| n/a | yes | -| [sqs\_workflow\_job\_queue](#input\_sqs\_workflow\_job\_queue) | SQS queue to monitor github events. | 
object({
    id  = string
    arn = string
  })
| `null` | no | | [ssm\_paths](#input\_ssm\_paths) | The root path used in SSM to store configuration and secrets. | 
object({
    root    = string
    webhook = string
  })
| n/a | yes | | [tags](#input\_tags) | Map of tags that will be added to created resources. By default resources will be tagged with name and environment. | `map(string)` | `{}` | no | | [tracing\_config](#input\_tracing\_config) | Configuration for lambda tracing. | 
object({
    mode                  = optional(string, null)
    capture_http_requests = optional(bool, false)
    capture_error         = optional(bool, false)
  })
| `{}` | no | diff --git a/modules/webhook/direct/README.md b/modules/webhook/direct/README.md index 2038623e8a..d050fe5395 100644 --- a/modules/webhook/direct/README.md +++ b/modules/webhook/direct/README.md @@ -28,7 +28,6 @@ No modules. | [aws_iam_role_policy.webhook_logging](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | | [aws_iam_role_policy.webhook_sqs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | | [aws_iam_role_policy.webhook_ssm](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | -| [aws_iam_role_policy.webhook_workflow_job_sqs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | | [aws_iam_role_policy.xray](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | | [aws_iam_role_policy_attachment.webhook_vpc_execution_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_lambda_function.webhook](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_function) | resource | @@ -41,7 +40,7 @@ No modules. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [config](#input\_config) | Configuration object for all variables. | 
object({
    prefix = string
    archive = optional(object({
      enable         = optional(bool, true)
      retention_days = optional(number, 7)
    }), {})
    tags = optional(map(string), {})

    lambda_subnet_ids         = optional(list(string), [])
    lambda_security_group_ids = optional(list(string), [])
    sqs_job_queues_arns       = list(string)
    sqs_workflow_job_queue = optional(object({
      id  = string
      arn = string
    }), null)
    lambda_zip                = optional(string, null)
    lambda_memory_size        = optional(number, 256)
    lambda_timeout            = optional(number, 10)
    role_permissions_boundary = optional(string, null)
    role_path                 = optional(string, null)
    logging_retention_in_days = optional(number, 180)
    logging_kms_key_id        = optional(string, null)
    lambda_s3_bucket          = optional(string, null)
    lambda_s3_key             = optional(string, null)
    lambda_s3_object_version  = optional(string, null)
    lambda_apigateway_access_log_settings = optional(object({
      destination_arn = string
      format          = string
    }), null)
    repository_white_list = optional(list(string), [])
    kms_key_arn           = optional(string, null)
    log_level             = optional(string, "info")
    lambda_runtime        = optional(string, "nodejs20.x")
    aws_partition         = optional(string, "aws")
    lambda_architecture   = optional(string, "arm64")
    github_app_parameters = object({
      webhook_secret = map(string)
    })
    tracing_config = optional(object({
      mode                  = optional(string, null)
      capture_http_requests = optional(bool, false)
      capture_error         = optional(bool, false)
    }), {})
    lambda_tags       = optional(map(string), {})
    api_gw_source_arn = string
    ssm_parameter_runner_matcher_config = object({
      name    = string
      arn     = string
      version = string
    })
  })
| n/a | yes | +| [config](#input\_config) | Configuration object for all variables. | 
object({
    prefix = string
    archive = optional(object({
      enable         = optional(bool, true)
      retention_days = optional(number, 7)
    }), {})
    tags = optional(map(string), {})

    lambda_subnet_ids         = optional(list(string), [])
    lambda_security_group_ids = optional(list(string), [])
    sqs_job_queues_arns       = list(string)
    lambda_zip                = optional(string, null)
    lambda_memory_size        = optional(number, 256)
    lambda_timeout            = optional(number, 10)
    role_permissions_boundary = optional(string, null)
    role_path                 = optional(string, null)
    logging_retention_in_days = optional(number, 180)
    logging_kms_key_id        = optional(string, null)
    lambda_s3_bucket          = optional(string, null)
    lambda_s3_key             = optional(string, null)
    lambda_s3_object_version  = optional(string, null)
    lambda_apigateway_access_log_settings = optional(object({
      destination_arn = string
      format          = string
    }), null)
    repository_white_list = optional(list(string), [])
    kms_key_arn           = optional(string, null)
    log_level             = optional(string, "info")
    lambda_runtime        = optional(string, "nodejs20.x")
    aws_partition         = optional(string, "aws")
    lambda_architecture   = optional(string, "arm64")
    github_app_parameters = object({
      webhook_secret = map(string)
    })
    tracing_config = optional(object({
      mode                  = optional(string, null)
      capture_http_requests = optional(bool, false)
      capture_error         = optional(bool, false)
    }), {})
    lambda_tags       = optional(map(string), {})
    api_gw_source_arn = string
    ssm_parameter_runner_matcher_config = object({
      name    = string
      arn     = string
      version = string
    })
  })
| n/a | yes | ## Outputs diff --git a/modules/webhook/direct/variables.tf b/modules/webhook/direct/variables.tf index dabad516a9..787b0e5666 100644 --- a/modules/webhook/direct/variables.tf +++ b/modules/webhook/direct/variables.tf @@ -11,10 +11,6 @@ variable "config" { lambda_subnet_ids = optional(list(string), []) lambda_security_group_ids = optional(list(string), []) sqs_job_queues_arns = list(string) - sqs_workflow_job_queue = optional(object({ - id = string - arn = string - }), null) lambda_zip = optional(string, null) lambda_memory_size = optional(number, 256) lambda_timeout = optional(number, 10) diff --git a/modules/webhook/direct/webhook.tf b/modules/webhook/direct/webhook.tf index 0bb9a88445..a8adc380a6 100644 --- a/modules/webhook/direct/webhook.tf +++ b/modules/webhook/direct/webhook.tf @@ -26,7 +26,6 @@ resource "aws_lambda_function" "webhook" { POWERTOOLS_TRACER_CAPTURE_ERROR = var.config.tracing_config.capture_error PARAMETER_GITHUB_APP_WEBHOOK_SECRET = var.config.github_app_parameters.webhook_secret.name REPOSITORY_ALLOW_LIST = jsonencode(var.config.repository_white_list) - SQS_WORKFLOW_JOB_QUEUE = try(var.config.sqs_workflow_job_queue.id, null) PARAMETER_RUNNER_MATCHER_CONFIG_PATH = var.config.ssm_parameter_runner_matcher_config.name PARAMETER_RUNNER_MATCHER_VERSION = var.config.ssm_parameter_runner_matcher_config.version # enforce cold start after Changes in SSM parameter } : k => v if v != null @@ -130,16 +129,6 @@ resource "aws_iam_role_policy" "webhook_kms" { }) } -resource "aws_iam_role_policy" "webhook_workflow_job_sqs" { - count = var.config.sqs_workflow_job_queue != null ? 1 : 0 - name = "publish-workflow-job-sqs-policy" - role = aws_iam_role.webhook_lambda.name - - policy = templatefile("${path.module}/../policies/lambda-publish-sqs-policy.json", { - sqs_resource_arns = jsonencode([var.config.sqs_workflow_job_queue.arn]) - }) -} - resource "aws_iam_role_policy" "webhook_ssm" { name = "publish-ssm-policy" role = aws_iam_role.webhook_lambda.name diff --git a/modules/webhook/eventbridge/README.md b/modules/webhook/eventbridge/README.md index ed83f40c8b..30913fb246 100644 --- a/modules/webhook/eventbridge/README.md +++ b/modules/webhook/eventbridge/README.md @@ -34,7 +34,6 @@ No modules. | [aws_iam_role_policy.dispatcher_logging](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | | [aws_iam_role_policy.dispatcher_sqs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | | [aws_iam_role_policy.dispatcher_ssm](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | -| [aws_iam_role_policy.dispatcher_workflow_job_sqs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | | [aws_iam_role_policy.dispatcher_xray](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | | [aws_iam_role_policy.webhook_eventbridge](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | | [aws_iam_role_policy.webhook_kms](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | @@ -55,7 +54,7 @@ No modules. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [config](#input\_config) | Configuration object for all variables. | 
object({
    prefix = string
    archive = optional(object({
      enable         = optional(bool, true)
      retention_days = optional(number, 7)
    }), {})
    tags = optional(map(string), {})

    lambda_subnet_ids         = optional(list(string), [])
    lambda_security_group_ids = optional(list(string), [])
    sqs_job_queues_arns       = list(string)
    sqs_workflow_job_queue = optional(object({
      id  = string
      arn = string
    }), null)
    lambda_zip                = optional(string, null)
    lambda_memory_size        = optional(number, 256)
    lambda_timeout            = optional(number, 10)
    role_permissions_boundary = optional(string, null)
    role_path                 = optional(string, null)
    logging_retention_in_days = optional(number, 180)
    logging_kms_key_id        = optional(string, null)
    lambda_s3_bucket          = optional(string, null)
    lambda_s3_key             = optional(string, null)
    lambda_s3_object_version  = optional(string, null)
    lambda_apigateway_access_log_settings = optional(object({
      destination_arn = string
      format          = string
    }), null)
    repository_white_list = optional(list(string), [])
    kms_key_arn           = optional(string, null)
    log_level             = optional(string, "info")
    lambda_runtime        = optional(string, "nodejs20.x")
    aws_partition         = optional(string, "aws")
    lambda_architecture   = optional(string, "arm64")
    github_app_parameters = object({
      webhook_secret = map(string)
    })
    tracing_config = optional(object({
      mode                  = optional(string, null)
      capture_http_requests = optional(bool, false)
      capture_error         = optional(bool, false)
    }), {})
    lambda_tags       = optional(map(string), {})
    api_gw_source_arn = string
    ssm_parameter_runner_matcher_config = object({
      name    = string
      arn     = string
      version = string
    })
    accept_events = optional(list(string), null)
  })
| n/a | yes | +| [config](#input\_config) | Configuration object for all variables. | 
object({
    prefix = string
    archive = optional(object({
      enable         = optional(bool, true)
      retention_days = optional(number, 7)
    }), {})
    tags = optional(map(string), {})

    lambda_subnet_ids         = optional(list(string), [])
    lambda_security_group_ids = optional(list(string), [])
    sqs_job_queues_arns       = list(string)
    lambda_zip                = optional(string, null)
    lambda_memory_size        = optional(number, 256)
    lambda_timeout            = optional(number, 10)
    role_permissions_boundary = optional(string, null)
    role_path                 = optional(string, null)
    logging_retention_in_days = optional(number, 180)
    logging_kms_key_id        = optional(string, null)
    lambda_s3_bucket          = optional(string, null)
    lambda_s3_key             = optional(string, null)
    lambda_s3_object_version  = optional(string, null)
    lambda_apigateway_access_log_settings = optional(object({
      destination_arn = string
      format          = string
    }), null)
    repository_white_list = optional(list(string), [])
    kms_key_arn           = optional(string, null)
    log_level             = optional(string, "info")
    lambda_runtime        = optional(string, "nodejs20.x")
    aws_partition         = optional(string, "aws")
    lambda_architecture   = optional(string, "arm64")
    github_app_parameters = object({
      webhook_secret = map(string)
    })
    tracing_config = optional(object({
      mode                  = optional(string, null)
      capture_http_requests = optional(bool, false)
      capture_error         = optional(bool, false)
    }), {})
    lambda_tags       = optional(map(string), {})
    api_gw_source_arn = string
    ssm_parameter_runner_matcher_config = object({
      name    = string
      arn     = string
      version = string
    })
    accept_events = optional(list(string), null)
  })
| n/a | yes | ## Outputs diff --git a/modules/webhook/eventbridge/dispatcher.tf b/modules/webhook/eventbridge/dispatcher.tf index b790897be5..15b76a127a 100644 --- a/modules/webhook/eventbridge/dispatcher.tf +++ b/modules/webhook/eventbridge/dispatcher.tf @@ -45,7 +45,6 @@ resource "aws_lambda_function" "dispatcher" { PARAMETER_RUNNER_MATCHER_CONFIG_PATH = var.config.ssm_parameter_runner_matcher_config.name PARAMETER_RUNNER_MATCHER_VERSION = var.config.ssm_parameter_runner_matcher_config.version # enforce cold start after Changes in SSM parameter REPOSITORY_ALLOW_LIST = jsonencode(var.config.repository_white_list) - SQS_WORKFLOW_JOB_QUEUE = try(var.config.sqs_workflow_job_queue.id, null) } : k => v if v != null } } @@ -138,13 +137,3 @@ resource "aws_iam_role_policy" "dispatcher_xray" { policy = data.aws_iam_policy_document.lambda_xray[0].json role = aws_iam_role.dispatcher_lambda.name } - -resource "aws_iam_role_policy" "dispatcher_workflow_job_sqs" { - count = var.config.sqs_workflow_job_queue != null ? 1 : 0 - name = "publish-workflow-job-sqs-policy" - role = aws_iam_role.dispatcher_lambda.name - - policy = templatefile("${path.module}/../policies/lambda-publish-sqs-policy.json", { - sqs_resource_arns = jsonencode([var.config.sqs_workflow_job_queue.arn]) - }) -} diff --git a/modules/webhook/eventbridge/variables.tf b/modules/webhook/eventbridge/variables.tf index 8980c6b5bc..0c6f435d70 100644 --- a/modules/webhook/eventbridge/variables.tf +++ b/modules/webhook/eventbridge/variables.tf @@ -11,10 +11,6 @@ variable "config" { lambda_subnet_ids = optional(list(string), []) lambda_security_group_ids = optional(list(string), []) sqs_job_queues_arns = list(string) - sqs_workflow_job_queue = optional(object({ - id = string - arn = string - }), null) lambda_zip = optional(string, null) lambda_memory_size = optional(number, 256) lambda_timeout = optional(number, 10) diff --git a/modules/webhook/variables.tf b/modules/webhook/variables.tf index 182f42053c..ba583d9859 100644 --- a/modules/webhook/variables.tf +++ b/modules/webhook/variables.tf @@ -40,14 +40,6 @@ variable "runner_matcher_config" { } } -variable "sqs_workflow_job_queue" { - description = "SQS queue to monitor github events." - type = object({ - id = string - arn = string - }) - default = null -} variable "lambda_zip" { description = "File location of the lambda zip file." type = string diff --git a/modules/webhook/webhook.tf b/modules/webhook/webhook.tf index 1d1e39a367..5f425bcef2 100644 --- a/modules/webhook/webhook.tf +++ b/modules/webhook/webhook.tf @@ -24,7 +24,6 @@ module "direct" { tags = var.tags, runner_matcher_config = var.runner_matcher_config, sqs_job_queues_arns = [for k, v in var.runner_matcher_config : v.arn] - sqs_workflow_job_queue = var.sqs_workflow_job_queue, lambda_zip = var.lambda_zip, lambda_memory_size = var.lambda_memory_size, lambda_timeout = var.lambda_timeout, @@ -61,7 +60,6 @@ module "eventbridge" { prefix = var.prefix, tags = var.tags, sqs_job_queues_arns = [for k, v in var.runner_matcher_config : v.arn] - sqs_workflow_job_queue = var.sqs_workflow_job_queue, lambda_zip = var.lambda_zip, lambda_memory_size = var.lambda_memory_size, lambda_timeout = var.lambda_timeout, diff --git a/outputs.tf b/outputs.tf index ce49e8927c..16235280af 100644 --- a/outputs.tf +++ b/outputs.tf @@ -51,9 +51,8 @@ output "ssm_parameters" { output "queues" { description = "SQS queues." value = { - build_queue_arn = aws_sqs_queue.queued_builds.arn - build_queue_dlq_arn = var.redrive_build_queue.enabled ? aws_sqs_queue.queued_builds_dlq[0].arn : null - webhook_workflow_job_queue = try(aws_sqs_queue.webhook_events_workflow_job_queue[*].arn, "") + build_queue_arn = aws_sqs_queue.queued_builds.arn + build_queue_dlq_arn = var.redrive_build_queue.enabled ? aws_sqs_queue.queued_builds_dlq[0].arn : null } } diff --git a/variables.tf b/variables.tf index 86535107af..f0b011873b 100644 --- a/variables.tf +++ b/variables.tf @@ -723,25 +723,6 @@ variable "lambda_architecture" { } } -variable "enable_workflow_job_events_queue" { - description = "Enabling this experimental feature will create a secondary SQS queue to which a copy of the workflow_job event will be delivered." - type = bool - default = false -} - -variable "workflow_job_queue_configuration" { - description = "Configuration options for workflow job queue which is only applicable if the flag enable_workflow_job_events_queue is set to true." - type = object({ - delay_seconds = number - visibility_timeout_seconds = number - message_retention_seconds = number - }) - default = { - "delay_seconds" : null, - "visibility_timeout_seconds" : null, - "message_retention_seconds" : null - } -} variable "enable_runner_binaries_syncer" { description = "Option to disable the lambda to sync GitHub runner distribution, useful when using a pre-build AMI." type = bool @@ -945,7 +926,6 @@ variable "job_retry" { default = {} } - variable "eventbridge" { description = <