feat: Add metric to track GitHub app rate limit (#4088)

## Description

This PR adds an optional metric to keep track of the remaining rate
limit for teh GItHub app.

## Notes

- Refactored the metric configuration to align the metric configuration
usages in all submodules. All changed are only impacting experimental
features. Which means non breaking.
- Refactored nameing gh-auth package, see separate commit.

---------

Co-authored-by: forest-pr|bot <forest-pr[bot]@users.noreply.github.com>
Co-authored-by: Stuart Pearson <[email protected]>

Author: 3 people

Diff for: README.md

@@ -147,7 +147,7 @@ Talk to the forestkeepers in the `runners-channel` on Slack.
 | <a name="input_enable_jit_config"></a> [enable\_jit\_config](#input\_enable\_jit\_config) | Overwrite the default behavior for JIT configuration. By default JIT configuration is enabled for ephemeral runners and disabled for non-ephemeral runners. In case of GHES check first if the JIT config API is avaialbe. In case you upgradeing from 3.x to 4.x you can set `enable_jit_config` to `false` to avoid a breaking change when having your own AMI. | `bool` | `null` | no |
 | <a name="input_enable_job_queued_check"></a> [enable\_job\_queued\_check](#input\_enable\_job\_queued\_check) | Only scale if the job event received by the scale up lambda is in the queued state. By default enabled for non ephemeral runners and disabled for ephemeral. Set this variable to overwrite the default behavior. | `bool` | `null` | no |
 | <a name="input_enable_managed_runner_security_group"></a> [enable\_managed\_runner\_security\_group](#input\_enable\_managed\_runner\_security\_group) | Enables creation of the default managed security group. Unmanaged security groups can be specified via `runner_additional_security_group_ids`. | `bool` | `true` | no |
-| <a name="input_enable_metrics_control_plane"></a> [enable\_metrics\_control\_plane](#input\_enable\_metrics\_control\_plane) | (Experimental) Enable or disable the metrics for the module. Feature can change or renamed without a major release. | `bool` | `false` | no |
+| <a name="input_enable_metrics_control_plane"></a> [enable\_metrics\_control\_plane](#input\_enable\_metrics\_control\_plane) | (Experimental) Enable or disable the metrics for the module. Feature can change or renamed without a major release. | `bool` | `null` | no |
 | <a name="input_enable_organization_runners"></a> [enable\_organization\_runners](#input\_enable\_organization\_runners) | Register runners to organization, instead of repo level | `bool` | `false` | no |
 | <a name="input_enable_runner_binaries_syncer"></a> [enable\_runner\_binaries\_syncer](#input\_enable\_runner\_binaries\_syncer) | Option to disable the lambda to sync GitHub runner distribution, useful when using a pre-build AMI. | `bool` | `true` | no |
 | <a name="input_enable_runner_detailed_monitoring"></a> [enable\_runner\_detailed\_monitoring](#input\_enable\_runner\_detailed\_monitoring) | Should detailed monitoring be enabled for the runner. Set this to true if you want to use detailed monitoring. See https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-cloudwatch-new.html for details. | `bool` | `false` | no |
@@ -165,7 +165,7 @@ Talk to the forestkeepers in the `runners-channel` on Slack.
 | <a name="input_instance_max_spot_price"></a> [instance\_max\_spot\_price](#input\_instance\_max\_spot\_price) | Max price price for spot instances per hour. This variable will be passed to the create fleet as max spot price for the fleet. | `string` | `null` | no |
 | <a name="input_instance_profile_path"></a> [instance\_profile\_path](#input\_instance\_profile\_path) | The path that will be added to the instance\_profile, if not set the environment name will be used. | `string` | `null` | no |
 | <a name="input_instance_target_capacity_type"></a> [instance\_target\_capacity\_type](#input\_instance\_target\_capacity\_type) | Default lifecycle used for runner instances, can be either `spot` or `on-demand`. | `string` | `"spot"` | no |
-| <a name="input_instance_termination_watcher"></a> [instance\_termination\_watcher](#input\_instance\_termination\_watcher) | Configuration for the instance termination watcher. This feature is Beta, changes will not trigger a major release as long in beta.<br><br>`enable`: Enable or disable the spot termination watcher.<br>`enable_metrics`: Enable or disable the metrics for the spot termination watcher.<br>`memory_size`: Memory size linit in MB of the lambda.<br>`s3_key`: S3 key for syncer lambda function. Required if using S3 bucket to specify lambdas.<br>`s3_object_version`: S3 object version for syncer lambda function. Useful if S3 versioning is enabled on source bucket.<br>`timeout`: Time out of the lambda in seconds.<br>`zip`: File location of the lambda zip file. | <pre>object({<br>    enable = optional(bool, false)<br>    enable_metric = optional(object({<br>      spot_warning = optional(bool, false)<br>    }))<br>    memory_size       = optional(number, null)<br>    s3_key            = optional(string, null)<br>    s3_object_version = optional(string, null)<br>    timeout           = optional(number, null)<br>    zip               = optional(string, null)<br>  })</pre> | `{}` | no |
+| <a name="input_instance_termination_watcher"></a> [instance\_termination\_watcher](#input\_instance\_termination\_watcher) | Configuration for the instance termination watcher. This feature is Beta, changes will not trigger a major release as long in beta.<br><br>`enable`: Enable or disable the spot termination watcher.<br>`memory_size`: Memory size linit in MB of the lambda.<br>`s3_key`: S3 key for syncer lambda function. Required if using S3 bucket to specify lambdas.<br>`s3_object_version`: S3 object version for syncer lambda function. Useful if S3 versioning is enabled on source bucket.<br>`timeout`: Time out of the lambda in seconds.<br>`zip`: File location of the lambda zip file. | <pre>object({<br>    enable            = optional(bool, false)<br>    enable_metric     = optional(string, null) # deprectaed<br>    memory_size       = optional(number, null)<br>    s3_key            = optional(string, null)<br>    s3_object_version = optional(string, null)<br>    timeout           = optional(number, null)<br>    zip               = optional(string, null)<br>  })</pre> | `{}` | no |
 | <a name="input_instance_types"></a> [instance\_types](#input\_instance\_types) | List of instance types for the action runner. Defaults are based on runner\_os (al2023 for linux and Windows Server Core for win). | `list(string)` | <pre>[<br>  "m5.large",<br>  "c5.large"<br>]</pre> | no |
 | <a name="input_job_queue_retention_in_seconds"></a> [job\_queue\_retention\_in\_seconds](#input\_job\_queue\_retention\_in\_seconds) | The number of seconds the job is held in the queue before it is purged. | `number` | `86400` | no |
 | <a name="input_job_retry"></a> [job\_retry](#input\_job\_retry) | Experimental! Can be removed / changed without trigger a major release.Configure job retries. The configuration enables job retries (for ephemeral runners). After creating the insances a message will be published to a job retry queue. The job retry check lambda is checking after a delay if the job is queued. If not the message will be published again on the scale-up (build queue). Using this feature can impact the reate limit of the GitHub app.<br><br>`enable`: Enable or disable the job retry feature.<br>`delay_in_seconds`: The delay in seconds before the job retry check lambda will check the job status.<br>`delay_backoff`: The backoff factor for the delay.<br>`lambda_memory_size`: Memory size limit in MB for the job retry check lambda.<br>`lambda_timeout`: Time out of the job retry check lambda in seconds.<br>`max_attempts`: The maximum number of attempts to retry the job. | <pre>object({<br>    enable             = optional(bool, false)<br>    delay_in_seconds   = optional(number, 300)<br>    delay_backoff      = optional(number, 2)<br>    lambda_memory_size = optional(number, 256)<br>    lambda_timeout     = optional(number, 30)<br>    max_attempts       = optional(number, 1)<br>  })</pre> | `{}` | no |
@@ -183,7 +183,8 @@ Talk to the forestkeepers in the `runners-channel` on Slack.
 | <a name="input_logging_kms_key_id"></a> [logging\_kms\_key\_id](#input\_logging\_kms\_key\_id) | Specifies the kms key id to encrypt the logs with. | `string` | `null` | no |
 | <a name="input_logging_retention_in_days"></a> [logging\_retention\_in\_days](#input\_logging\_retention\_in\_days) | Specifies the number of days you want to retain log events for the lambda log group. Possible values are: 0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, and 3653. | `number` | `180` | no |
 | <a name="input_matcher_config_parameter_store_tier"></a> [matcher\_config\_parameter\_store\_tier](#input\_matcher\_config\_parameter\_store\_tier) | The tier of the parameter store for the matcher configuration. Valid values are `Standard`, and `Advanced`. | `string` | `"Standard"` | no |
-| <a name="input_metrics_namespace"></a> [metrics\_namespace](#input\_metrics\_namespace) | The namespace for the metrics created by the module. Merics will only be created if explicit enabled. | `string` | `"GitHub Runners"` | no |
+| <a name="input_metrics"></a> [metrics](#input\_metrics) | Configuration for metrics created by the module, by default disabled to avoid additional costs. When metrics are enable all metrics are created unless explicit configured otherwise. | <pre>object({<br>    enable    = optional(bool, false)<br>    namespace = optional(string, "GitHub Runners")<br>    metric = optional(object({<br>      enable_github_app_rate_limit    = optional(bool, true)<br>      enable_job_retry                = optional(bool, true)<br>      enable_spot_termination_warning = optional(bool, true)<br>    }), {})<br>  })</pre> | `{}` | no |
+| <a name="input_metrics_namespace"></a> [metrics\_namespace](#input\_metrics\_namespace) | The namespace for the metrics created by the module. Merics will only be created if explicit enabled. | `string` | `null` | no |
 | <a name="input_minimum_running_time_in_minutes"></a> [minimum\_running\_time\_in\_minutes](#input\_minimum\_running\_time\_in\_minutes) | The time an ec2 action runner should be running at minimum before terminated, if not busy. | `number` | `null` | no |
 | <a name="input_pool_config"></a> [pool\_config](#input\_pool\_config) | The configuration for updating the pool. The `pool_size` to adjust to by the events triggered by the `schedule_expression`. For example you can configure a cron expression for weekdays to adjust the pool to 10 and another expression for the weekend to adjust the pool to 1. Use `schedule_expression_timezone` to override the schedule time zone (defaults to UTC). | <pre>list(object({<br>    schedule_expression          = string<br>    schedule_expression_timezone = optional(string)<br>    size                         = number<br>  }))</pre> | `[]` | no |
 | <a name="input_pool_lambda_memory_size"></a> [pool\_lambda\_memory\_size](#input\_pool\_lambda\_memory\_size) | Memory size limit for scale-up lambda. | `number` | `512` | no |

Diff for: docs/configuration.md

@@ -191,6 +191,15 @@ This feature has been disabled by default.
 
 The watcher will act on all spot termination notificatins and log all onses relevant to the runner module. Therefor we suggest to only deploy the watcher once. You can either deploy the watcher by enabling in one of your deployments or deploy the watcher as a stand alone module.
 
+## Metrics
+
+The module supports metrics (experimental feature) to monitor the system. The metrics are disabled by default. To enable the metrics set `metrics.enable = true`. If set to true, all module managed metrics are used, you can configure the one by one via the `metrics` object. The metrics are created in the namespace `GitHub Runners`.
+
+### Supported metrics
+
+- **GitHubAppRateLimitRemaining**: Remaining rate limit for the GitHub App.
+- **JobRetry**: Number of job retries, only relevant when job retry is enabled.
+- **SpotInterruptionWarning**: Number of spot interruption warnings received by the termination watcher, only relevant when the termination watcher is enabled.
 
 ## Debugging
 

Diff for: examples/default/main.tf

@@ -114,21 +114,25 @@ module "runners" {
 
   instance_termination_watcher = {
     enable = true
-    enable_metric = {
-      spot_warning = true
-    }
   }
 
-  # enable job_retry feature. Be careful with this feature, it can lead to API rate limits.
+  # enable metric creation  (experimental)
+  # metrics = {
+  #   enable = true
+  #   metric = {
+  #     enable_spot_termination_warning = true
+  #     enable_job_retry                = false
+  #     enable_github_app_rate_limit    = true
+  #   }
+  # }
+
+  # enable job_retry feature. Be careful with this feature, it can lead to you hitting API rate limits.
   # job_retry = {
   #   enable           = true
   #   max_attempts     = 1
   #   delay_in_seconds = 180
   # }
 
-  # enable metric creation by the control plane (experimental)
-  # enable_metrics_control_plane = true
-
   # enable CMK instead of aws managed key for encryptions
   # kms_key_arn = aws_kms_key.github.arn
 }

Diff for: examples/multi-runner/main.tf

@@ -103,8 +103,15 @@ module "runners" {
   # Enable to track the spot instance termination warning
   # instance_termination_watcher = {
   #   enable         = true
-  #   enable_metric = {
-  #     spot_warning = true
+  # }
+
+  # Enable metrics
+  # metrics = {
+  #   enable = true
+  #   metric = {
+  #     enable_github_app_rate_limit    = true
+  #     enable_job_retry                = false
+  #     enable_spot_termination_warning = true
   #   }
   # }
 }

Diff for: examples/termination-watcher/main.tf

@@ -2,12 +2,15 @@ module "spot_termination_watchter" {
   source = "../../modules/termination-watcher"
 
   config = {
-    enable_metric = {
-      spot_warning = true
+    metrics = {
+      enable = true
+      metric = {
+        enable_spot_termination_warning = true
+      }
     }
     prefix = "global"
     tag_filters = {
       "ghr:Application" = "github-action-runner"
     }
   }
-}
+}

Diff for: lambdas/functions/control-plane/jest.config.ts

@@ -6,10 +6,10 @@ const config: Config = {
   ...defaultConfig,
   coverageThreshold: {
     global: {
-      statements: 97.78,
-      branches: 96.61,
-      functions: 95.84,
-      lines: 97.71,
+      statements: 97.86,
+      branches: 96.68,
+      functions: 95.95,
+      lines: 97.8,
     },
   },
 };

Diff for: lambdas/functions/control-plane/package.json

@@ -38,6 +38,7 @@
     "ts-node-dev": "^2.0.0"
   },
   "dependencies": {
+    "@aws-lambda-powertools/parameters": "^2.7.0",
     "@aws-sdk/client-ec2": "^3.637.0",
     "@aws-sdk/client-sqs": "^3.637.0",
     "@aws-sdk/types": "^3.609.0",

Diff for: lambdas/functions/control-plane/src/gh-auth/gh-auth.test.ts renamed to lambdas/functions/control-plane/src/github/auth.test.ts

@@ -7,7 +7,7 @@ import { mocked } from 'jest-mock';
 import { MockProxy, mock } from 'jest-mock-extended';
 import nock from 'nock';
 
-import { createGithubAppAuth, createOctokitClient } from './gh-auth';
+import { createGithubAppAuth, createOctokitClient } from './auth';
 
 jest.mock('@terraform-aws-github-runner/aws-ssm-util');
 jest.mock('@octokit/auth-app');

Diff for: lambdas/functions/control-plane/src/gh-auth/gh-auth.ts renamed to lambdas/functions/control-plane/src/github/auth.ts

@@ -1,6 +1,6 @@
 import { Octokit } from '@octokit/rest';
 import { ActionRequestMessage } from '../scale-runners/scale-up';
-import { getOctokit } from './gh-octokit';
+import { getOctokit } from './octokit';
 
 const mockOctokit = {
   apps: {
@@ -9,7 +9,7 @@ const mockOctokit = {
   },
 };
 
-jest.mock('../gh-auth/gh-auth', () => ({
+jest.mock('../github/auth', () => ({
   createGithubInstallationAuth: jest.fn().mockImplementation(async (installationId) => {
     return { token: 'token', type: 'installation', installationId: installationId };
   }),
@@ -21,7 +21,7 @@ jest.mock('@octokit/rest', () => ({
   Octokit: jest.fn().mockImplementation(() => mockOctokit),
 }));
 
-jest.mock('../gh-auth/gh-auth');
+jest.mock('../github/auth');
 
 describe('Test getOctokit', () => {
   const data = [

Diff for: lambdas/functions/control-plane/src/gh-auth/gh-octokit.test.ts renamed to lambdas/functions/control-plane/src/github/octokit.test.ts

@@ -1,6 +1,6 @@
 import { Octokit } from '@octokit/rest';
 import { ActionRequestMessage } from '../scale-runners/scale-up';
-import { createGithubAppAuth, createGithubInstallationAuth, createOctokitClient } from './gh-auth';
+import { createGithubAppAuth, createGithubInstallationAuth, createOctokitClient } from './auth';
 
 export async function getInstallationId(
   ghesApiUrl: string,

Diff for: lambdas/functions/control-plane/src/gh-auth/gh-octokit.ts renamed to lambdas/functions/control-plane/src/github/octokit.ts

@@ -0,0 +1,70 @@
+import { ResponseHeaders } from '@octokit/types';
+import { createSingleMetric } from '@terraform-aws-github-runner/aws-powertools-util';
+import { MetricUnit } from '@aws-lambda-powertools/metrics';
+import { metricGitHubAppRateLimit } from './rate-limit';
+
+process.env.PARAMETER_GITHUB_APP_ID_NAME = 'test';
+jest.mock('@terraform-aws-github-runner/aws-ssm-util', () => ({
+  ...jest.requireActual('@terraform-aws-github-runner/aws-ssm-util'),
+  // get parameter name from process.env.PARAMETER_GITHUB_APP_ID_NAME rerunt 1234
+  getParameter: jest.fn((name: string) => {
+    if (name === process.env.PARAMETER_GITHUB_APP_ID_NAME) {
+      return '1234';
+    } else {
+      return '';
+    }
+  }),
+}));
+
+jest.mock('@terraform-aws-github-runner/aws-powertools-util', () => ({
+  ...jest.requireActual('@terraform-aws-github-runner/aws-powertools-util'),
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  createSingleMetric: jest.fn((name: string, unit: string, value: number, dimensions?: Record<string, string>) => {
+    return {
+      addMetadata: jest.fn(),
+    };
+  }),
+}));
+
+describe('metricGitHubAppRateLimit', () => {
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+
+  it('should update rate limit metric', async () => {
+    // set process.env.ENABLE_METRIC_GITHUB_APP_RATE_LIMIT to true
+    process.env.ENABLE_METRIC_GITHUB_APP_RATE_LIMIT = 'true';
+    const headers: ResponseHeaders = {
+      'x-ratelimit-remaining': '10',
+      'x-ratelimit-limit': '60',
+    };
+
+    await metricGitHubAppRateLimit(headers);
+
+    expect(createSingleMetric).toHaveBeenCalledWith('GitHubAppRateLimitRemaining', MetricUnit.Count, 10, {
+      AppId: '1234',
+    });
+  });
+
+  it('should not update rate limit metric', async () => {
+    // set process.env.ENABLE_METRIC_GITHUB_APP_RATE_LIMIT to false
+    process.env.ENABLE_METRIC_GITHUB_APP_RATE_LIMIT = 'false';
+    const headers: ResponseHeaders = {
+      'x-ratelimit-remaining': '10',
+      'x-ratelimit-limit': '60',
+    };
+
+    await metricGitHubAppRateLimit(headers);
+
+    expect(createSingleMetric).not.toHaveBeenCalled();
+  });
+
+  it('should not update rate limit metric if headers are undefined', async () => {
+    // set process.env.ENABLE_METRIC_GITHUB_APP_RATE_LIMIT to true
+    process.env.ENABLE_METRIC_GITHUB_APP_RATE_LIMIT = 'true';
+
+    await metricGitHubAppRateLimit(undefined as unknown as ResponseHeaders);
+
+    expect(createSingleMetric).not.toHaveBeenCalled();
+  });
+});

Diff for: lambdas/functions/control-plane/src/github/rate-limit.test.ts

@@ -0,0 +1,25 @@
+import { ResponseHeaders } from '@octokit/types';
+import { createSingleMetric, logger } from '@terraform-aws-github-runner/aws-powertools-util';
+import { MetricUnit } from '@aws-lambda-powertools/metrics';
+import yn from 'yn';
+import { getParameter } from '@terraform-aws-github-runner/aws-ssm-util';
+
+export async function metricGitHubAppRateLimit(headers: ResponseHeaders): Promise<void> {
+  try {
+    const remaining = parseInt(headers['x-ratelimit-remaining'] as string);
+    const limit = parseInt(headers['x-ratelimit-limit'] as string);
+
+    logger.debug(`Rate limit remaining: ${remaining}, limit: ${limit}`);
+
+    const updateMetric = yn(process.env.ENABLE_METRIC_GITHUB_APP_RATE_LIMIT);
+    if (updateMetric) {
+      const appId = await getParameter(process.env.PARAMETER_GITHUB_APP_ID_NAME);
+      const metric = createSingleMetric('GitHubAppRateLimitRemaining', MetricUnit.Count, remaining, {
+        AppId: appId,
+      });
+      metric.addMetadata('AppId', appId);
+    }
+  } catch (e) {
+    logger.debug(`Error updating rate limit metric`, { error: e });
+  }
+}

Diff for: lambdas/functions/control-plane/src/github/rate-limit.ts

@@ -1,6 +1,7 @@
 declare namespace NodeJS {
   export interface ProcessEnv {
     AWS_REGION: string;
+    ENABLE_METRIC_GITHUB_APP_RATE_LIMIT: string;
     ENABLE_ON_DEMAND_FAILOVER_FOR_ERRORS: string;
     ENVIRONMENT: string;
     GHES_URL: string;

Diff for: lambdas/functions/control-plane/src/modules.d.ts

@@ -4,7 +4,7 @@ import moment from 'moment-timezone';
 import nock from 'nock';
 
 import { listEC2Runners } from '../aws/runners';
-import * as ghAuth from '../gh-auth/gh-auth';
+import * as ghAuth from '../github/auth';
 import { createRunners } from '../scale-runners/scale-up';
 import { adjust } from './pool';
 
@@ -27,7 +27,7 @@ jest.mock('./../aws/runners', () => ({
   ...jest.requireActual('./../aws/runners'),
   listEC2Runners: jest.fn(),
 }));
-jest.mock('./../gh-auth/gh-auth');
+jest.mock('./../github/auth');
 jest.mock('./../scale-runners/scale-up');
 
 const mocktokit = Octokit as jest.MockedClass<typeof Octokit>;