fix: Remove fifo queues

Author: npalm

examples/arm64/main.tf

@@ -80,9 +80,6 @@ module "runners" {
   delay_webhook_event   = 5
   runners_maximum_count = 1
 
-  # set up a fifo queue to remain order
-  enable_fifo_build_queue = true
-
   # override scaling down
   scale_down_schedule_expression = "cron(* * * * ? *)"
 }

examples/multi-runner/main.tf

@@ -47,7 +47,6 @@ module "runners" {
   #        labelMatchers = [["self-hosted", "linux", "x64", "amazon"]]
   #        exactMatch    = false
   #      }
-  #      fifo                = true
   #      delay_webhook_event = 0
   #      runner_config = {
   #        runner_os                       = "linux"

lambdas/functions/control-plane/src/local.ts

@@ -28,7 +28,7 @@ const sqsEvent = {
       messageAttributes: {},
       md5OfBody: '4aef3bd70526e152e86426a0938cbec6',
       eventSource: 'aws:sqs',
-      eventSourceARN: 'arn:aws:sqs:us-west-2:916370655143:cicddev-queued-builds.fifo',
+      eventSourceARN: 'arn:aws:sqs:us-west-2:916370655143:cicddev-queued-builds',
       awsRegion: 'us-west-2',
     },
   ],

lambdas/functions/webhook/jest.config.ts

@@ -6,10 +6,10 @@ const config: Config = {
   ...defaultConfig,
   coverageThreshold: {
     global: {
-      statements: 99.13,
+      statements: 99.19,
       branches: 96.87,
       functions: 100,
-      lines: 99.09,
+      lines: 99.16,
     },
   },
 };

lambdas/functions/webhook/src/sqs/index.test.ts

@@ -27,49 +27,26 @@ describe('Test sending message to SQS.', () => {
     repositoryName: 'test',
     repositoryOwner: 'owner',
     queueId: queueUrl,
-    queueFifo: false,
     repoOwnerType: 'Organization',
   };
 
   afterEach(() => {
     jest.clearAllMocks();
   });
 
-  it('no fifo queue', async () => {
+  it('should queue a message', async () => {
     // Arrange
-    const no_fifo_message: ActionRequestMessage = {
-      ...message,
-      queueFifo: false,
-    };
     const sqsMessage: SendMessageCommandInput = {
       QueueUrl: queueUrl,
-      MessageBody: JSON.stringify(no_fifo_message),
+      MessageBody: JSON.stringify(message),
     };
     // Act
-    const result = await sendActionRequest(no_fifo_message);
+    const result = await sendActionRequest(message);
 
     // Assert
     expect(mockSQS.sendMessage).toBeCalledWith(sqsMessage);
     expect(result).resolves;
   });
-
-  it('use a fifo queue', async () => {
-    // Arrange
-    const fifo_message: ActionRequestMessage = {
-      ...message,
-      queueFifo: true,
-    };
-    const sqsMessage: SendMessageCommandInput = {
-      QueueUrl: queueUrl,
-      MessageBody: JSON.stringify(fifo_message),
-    };
-    // Act
-    const result = await sendActionRequest(fifo_message);
-
-    // Assert
-    expect(mockSQS.sendMessage).toBeCalledWith({ ...sqsMessage, MessageGroupId: String(message.id) });
-    expect(result).resolves;
-  });
 });
 
 describe('Test sending message to SQS.', () => {

lambdas/functions/webhook/src/sqs/index.ts

@@ -12,7 +12,6 @@ export interface ActionRequestMessage {
   repositoryOwner: string;
   installationId: number;
   queueId: string;
-  queueFifo: boolean;
   repoOwnerType: string;
 }
 
@@ -27,7 +26,6 @@ export interface RunnerMatcherConfig {
   matcherConfig: MatcherConfig;
   id: string;
   arn: string;
-  fifo: boolean;
 }
 
 export interface GithubWorkflowEvent {
@@ -43,9 +41,6 @@ export const sendActionRequest = async (message: ActionRequestMessage): Promise<
   };
 
   logger.debug(`sending message to SQS: ${JSON.stringify(sqsMessage)}`);
-  if (message.queueFifo) {
-    sqsMessage.MessageGroupId = String(message.id);
-  }
 
   await sqs.sendMessage(sqsMessage);
 };

lambdas/functions/webhook/src/webhook/index.test.ts

@@ -449,7 +449,6 @@ describe('handler', () => {
         eventType: 'workflow_job',
         installationId: 0,
         queueId: 'ubuntu-queue-id',
-        queueFifo: false,
         repoOwnerType: 'Organization',
       });
     });
@@ -492,7 +491,6 @@ describe('handler', () => {
         eventType: 'workflow_job',
         installationId: 0,
         queueId: 'ubuntu-queue-id',
-        queueFifo: false,
         repoOwnerType: 'Organization',
       });
     });
@@ -532,7 +530,6 @@ describe('handler', () => {
       eventType: 'workflow_job',
       installationId: 0,
       queueId: 'ubuntu-queue-id',
-      queueFifo: false,
       repoOwnerType: 'Organization',
     });
   });

lambdas/functions/webhook/src/webhook/index.ts

@@ -52,7 +52,6 @@ async function handleWorkflowJob(
           eventType: githubEvent,
           installationId: installationId,
           queueId: queue.id,
-          queueFifo: queue.fifo,
           repoOwnerType: body.repository.owner.type,
         });
         logger.info(`Successfully queued job for ${body.repository.full_name} to the queue ${queue.id}`);

main.tf

@@ -58,7 +58,7 @@ resource "aws_sqs_queue_policy" "webhook_events_workflow_job_queue_policy" {
 }
 
 resource "aws_sqs_queue" "queued_builds" {
-  name                        = "${var.prefix}-queued-builds${var.enable_fifo_build_queue ? ".fifo" : ""}"
+  name                        = "${var.prefix}-queued-builds"
   delay_seconds               = var.delay_webhook_event
   visibility_timeout_seconds  = var.runners_scale_up_lambda_timeout
   message_retention_seconds   = var.job_queue_retention_in_seconds
@@ -103,7 +103,7 @@ resource "aws_sqs_queue_policy" "build_queue_dlq_policy" {
 
 resource "aws_sqs_queue" "queued_builds_dlq" {
   count = var.redrive_build_queue.enabled ? 1 : 0
-  name  = "${var.prefix}-queued-builds_dead_letter${var.enable_fifo_build_queue ? ".fifo" : ""}"
+  name  = "${var.prefix}-queued-builds_dead_letter"
 
   sqs_managed_sse_enabled           = var.queue_encryption.sqs_managed_sse_enabled
   kms_master_key_id                 = var.queue_encryption.kms_master_key_id
@@ -135,7 +135,6 @@ module "webhook" {
     (aws_sqs_queue.queued_builds.id) = {
       id : aws_sqs_queue.queued_builds.id
       arn : aws_sqs_queue.queued_builds.arn
-      fifo : var.enable_fifo_build_queue
       matcherConfig : {
         labelMatchers : [local.runner_labels]
         exactMatch : var.enable_runner_workflow_job_labels_check_all

modules/multi-runner/queues.tf

@@ -27,14 +27,12 @@ data "aws_iam_policy_document" "deny_unsecure_transport" {
 }
 
 resource "aws_sqs_queue" "queued_builds" {
-  for_each                    = var.multi_runner_config
-  name                        = "${var.prefix}-${each.key}-queued-builds${each.value.fifo ? ".fifo" : ""}"
-  delay_seconds               = each.value.runner_config.delay_webhook_event
-  visibility_timeout_seconds  = var.runners_scale_up_lambda_timeout
-  message_retention_seconds   = each.value.runner_config.job_queue_retention_in_seconds
-  fifo_queue                  = each.value.fifo
-  receive_wait_time_seconds   = 0
-  content_based_deduplication = each.value.fifo
+  for_each                   = var.multi_runner_config
+  name                       = "${var.prefix}-${each.key}-queued-builds"
+  delay_seconds              = each.value.runner_config.delay_webhook_event
+  visibility_timeout_seconds = var.runners_scale_up_lambda_timeout
+  message_retention_seconds  = each.value.runner_config.job_queue_retention_in_seconds
+  receive_wait_time_seconds  = 0
   redrive_policy = each.value.redrive_build_queue.enabled ? jsonencode({
     deadLetterTargetArn = aws_sqs_queue.queued_builds_dlq[each.key].arn,
     maxReceiveCount     = each.value.redrive_build_queue.maxReceiveCount
@@ -55,12 +53,11 @@ resource "aws_sqs_queue_policy" "build_queue_policy" {
 
 resource "aws_sqs_queue" "queued_builds_dlq" {
   for_each = { for config, values in var.multi_runner_config : config => values if values.redrive_build_queue.enabled }
-  name     = "${var.prefix}-${each.key}-queued-builds_dead_letter${each.value.fifo ? ".fifo" : ""}"
+  name     = "${var.prefix}-${each.key}-queued-builds_dead_letter"
 
   sqs_managed_sse_enabled           = var.queue_encryption.sqs_managed_sse_enabled
   kms_master_key_id                 = var.queue_encryption.kms_master_key_id
   kms_data_key_reuse_period_seconds = var.queue_encryption.kms_data_key_reuse_period_seconds
-  fifo_queue                        = each.value.fifo
   tags                              = var.tags
 }
 

modules/multi-runner/variables.tf

@@ -125,7 +125,6 @@ variable "multi_runner_config" {
       exactMatch    = optional(bool, false)
       priority      = optional(number, 999)
     })
-    fifo = optional(bool, false)
     redrive_build_queue = optional(object({
       enabled         = bool
       maxReceiveCount = number

modules/webhook/variables.tf

@@ -25,9 +25,8 @@ variable "tags" {
 variable "runner_matcher_config" {
   description = "SQS queue to publish accepted build events based on the runner type. When exact match is disabled the webhook accepts the event if one of the workflow job labels is part of the matcher. The priority defines the order the matchers are applied."
   type = map(object({
-    arn  = string
-    id   = string
-    fifo = bool
+    arn = string
+    id  = string
     matcherConfig = object({
       labelMatchers = list(list(string))
       exactMatch    = bool

variables.tf

@@ -630,9 +630,14 @@ variable "lambda_principals" {
 }
 
 variable "enable_fifo_build_queue" {
-  description = "Enable a FIFO queue to keep the order of events received by the webhook. Recommended for repo level runners."
+  description = "(FEATURE REMOVED) Enable a FIFO queue to keep the order of events received by the webhook. Recommended for repo level runners."
   type        = bool
   default     = false
+
+  validation {
+    condition     = var.enable_fifo_build_queue == false
+    error_message = "The feature for FIFO build queue is not supported anymore."
+  }
 }
 
 variable "redrive_build_queue" {