feat: Allow to disable runner max scaling check (#3849)

## Description
Allow to disable to maximum runner check when max runners is set to -1

## Background
The maximum check is calling the AWS API describeInstances. When having
a huge fleet of runners this call is expensive. Currently we have an
issue with this API, this option allows to avoid calling the
describeInstance API. Drawback is there will be nox safegouard for
scaling.

---------

Co-authored-by: forest-pr|bot <forest-pr[bot]@users.noreply.github.com>

Author: npalm

Diff for: lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts

@@ -241,6 +241,14 @@ describe('scaleUp with GHES', () => {
       expect(mockOctokit.actions.createRegistrationTokenForRepo).not.toBeCalled();
     });
 
+    it('does create a runner if maximum is set to -1', async () => {
+      process.env.RUNNERS_MAXIMUM_COUNT = '-1';
+      process.env.ENABLE_EPHEMERAL_RUNNERS = 'false';
+      await scaleUpModule.scaleUp('aws:sqs', TEST_DATA);
+      expect(listEC2Runners).not.toHaveBeenCalled();
+      expect(createRunner).toHaveBeenCalled();
+    });
+
     it('creates a token when maximum runners has not been reached', async () => {
       process.env.ENABLE_EPHEMERAL_RUNNERS = 'false';
       await scaleUpModule.scaleUp('aws:sqs', TEST_DATA);

Diff for: lambdas/functions/control-plane/src/scale-runners/scale-up.ts

@@ -277,14 +277,18 @@ export async function scaleUp(eventSource: string, payload: ActionRequestMessage
   const ghAuth = await createGithubInstallationAuth(installationId, ghesApiUrl);
   const githubInstallationClient = await createOctoClient(ghAuth.token, ghesApiUrl);
   if (!enableJobQueuedCheck || (await isJobQueued(githubInstallationClient, payload))) {
-    const currentRunners = await listEC2Runners({
-      environment,
-      runnerType,
-      runnerOwner,
-    });
-    logger.info(`Current runners: ${currentRunners.length} of ${maximumRunners}`);
+    let scaleUp = true;
+    if (maximumRunners !== -1) {
+      const currentRunners = await listEC2Runners({
+        environment,
+        runnerType,
+        runnerOwner,
+      });
+      logger.info(`Current runners: ${currentRunners.length} of ${maximumRunners}`);
+      scaleUp = currentRunners.length < maximumRunners;
+    }
 
-    if (currentRunners.length < maximumRunners) {
+    if (scaleUp) {
       logger.info(`Attempting to launch a new runner`);
 
       await createRunners(

Diff for: modules/multi-runner/README.md

@@ -160,7 +160,7 @@ variable "multi_runner_config" {
         runner_group_name: "Name of the runner group."
         runner_name_prefix: "Prefix for the GitHub runner name."
         runner_run_as: "Run the GitHub actions agent as user."
-        runners_maximum_count: "The maximum number of runners that will be created."
+        runners_maximum_count: "The maximum number of runners that will be created. Setting the variable to `-1` desiables the maximum check."
         scale_down_schedule_expression: "Scheduler expression to check every x for scale down."
         scale_up_reserved_concurrent_executions: "Amount of reserved concurrent executions for the scale-up lambda function. A value of 0 disables lambda from being triggered and -1 removes any concurrency limitations."
         userdata_template: "Alternative user-data template, replacing the default template. By providing your own user_data you have to take care of installing all required software, including the action runner. Variables userdata_pre/post_install are ignored."

Diff for: modules/multi-runner/variables.tf

@@ -204,7 +204,7 @@ yarn run dist
 | <a name="input_runner_run_as"></a> [runner\_run\_as](#input\_runner\_run\_as) | Run the GitHub actions agent as user. | `string` | `"ec2-user"` | no |
 | <a name="input_runners_lambda_s3_key"></a> [runners\_lambda\_s3\_key](#input\_runners\_lambda\_s3\_key) | S3 key for runners lambda function. Required if using S3 bucket to specify lambdas. | `string` | `null` | no |
 | <a name="input_runners_lambda_s3_object_version"></a> [runners\_lambda\_s3\_object\_version](#input\_runners\_lambda\_s3\_object\_version) | S3 object version for runners lambda function. Useful if S3 versioning is enabled on source bucket. | `string` | `null` | no |
-| <a name="input_runners_maximum_count"></a> [runners\_maximum\_count](#input\_runners\_maximum\_count) | The maximum number of runners that will be created. | `number` | `3` | no |
+| <a name="input_runners_maximum_count"></a> [runners\_maximum\_count](#input\_runners\_maximum\_count) | The maximum number of runners that will be created. Setting the variable to `-1` desiables the maximum check. | `number` | `3` | no |
 | <a name="input_s3_runner_binaries"></a> [s3\_runner\_binaries](#input\_s3\_runner\_binaries) | Bucket details for cached GitHub binary. | <pre>object({<br>    arn = string<br>    id  = string<br>    key = string<br>  })</pre> | n/a | yes |
 | <a name="input_scale_down_schedule_expression"></a> [scale\_down\_schedule\_expression](#input\_scale\_down\_schedule\_expression) | Scheduler expression to check every x for scale down. | `string` | `"cron(*/5 * * * ? *)"` | no |
 | <a name="input_scale_up_reserved_concurrent_executions"></a> [scale\_up\_reserved\_concurrent\_executions](#input\_scale\_up\_reserved\_concurrent\_executions) | Amount of reserved concurrent executions for the scale-up lambda function. A value of 0 disables lambda from being triggered and -1 removes any concurrency limitations. | `number` | `1` | no |

Diff for: modules/runners/README.md

@@ -288,7 +288,7 @@ variable "runner_run_as" {
 }
 
 variable "runners_maximum_count" {
-  description = "The maximum number of runners that will be created."
+  description = "The maximum number of runners that will be created. Setting the variable to `-1` desiables the maximum check."
   type        = number
   default     = 3
 }