feat: support AWS EventBridge

README.md

@@ -250,6 +250,7 @@ Talk to the forestkeepers in the `runners-channel` on Slack.
 | <a name="input_webhook_lambda_s3_object_version"></a> [webhook\_lambda\_s3\_object\_version](#input\_webhook\_lambda\_s3\_object\_version) | S3 object version for webhook lambda function. Useful if S3 versioning is enabled on source bucket. | `string` | `null` | no |
 | <a name="input_webhook_lambda_timeout"></a> [webhook\_lambda\_timeout](#input\_webhook\_lambda\_timeout) | Time out of the webhook lambda in seconds. | `number` | `10` | no |
 | <a name="input_webhook_lambda_zip"></a> [webhook\_lambda\_zip](#input\_webhook\_lambda\_zip) | File location of the webhook lambda zip file. | `string` | `null` | no |
+| <a name="input_webhook_mode"></a> [webhook\_mode](#input\_webhook\_mode) | The webhook and dispatching to runner queues supports two modes. Direct messages, are delivered directly to the runner queues. EventBridge messages are delivered to an EventBridge bus and then dispatched to the runner queues. Valid values are `direct` and `eventbridge`. | `string` | `"direct"` | no |
 | <a name="input_workflow_job_queue_configuration"></a> [workflow\_job\_queue\_configuration](#input\_workflow\_job\_queue\_configuration) | Configuration options for workflow job queue which is only applicable if the flag enable\_workflow\_job\_events\_queue is set to true. | <pre>object({<br/>    delay_seconds              = number<br/>    visibility_timeout_seconds = number<br/>    message_retention_seconds  = number<br/>  })</pre> | <pre>{<br/>  "delay_seconds": null,<br/>  "message_retention_seconds": null,<br/>  "visibility_timeout_seconds": null<br/>}</pre> | no |
 
 ## Outputs

examples/default/main.tf

@@ -97,8 +97,17 @@ module "runners" {
   # prefix GitHub runners with the environment name
   runner_name_prefix = "${local.environment}_"
 
+  # webhook supports to modes, either direct or via the eventbridge
+  webhook_mode = "eventbridge" # or "direct"
+
   # Enable debug logging for the lambda functions
-  log_level = "info"
+  log_level = "debug"
+
+  tracing_config = {
+    mode                  = "Active"
+    capture_error         = true
+    capture_http_requests = true
+  }
 
   enable_ami_housekeeper = true
   ami_housekeeper_cleanup_config = {
@@ -158,3 +167,7 @@ module "webhook_github_app" {
 #   name          = "alias/github/action-runners"
 #   target_key_id = aws_kms_key.github.key_id
 # }
+# moved {
+#   from = module.runners.module.webhook.aws_lambda_function.webhook
+#   to   = module.runners.module.webhook.module.webhook.aws_lambda_function.webhook
+# }

lambdas/functions/ami-housekeeper/package.json

@@ -41,7 +41,7 @@
     "@aws-github-runner/aws-ssm-util": "*",
     "@aws-sdk/client-ec2": "^3.670.0",
     "@aws-sdk/client-ssm": "^3.670.0",
-    "@aws-sdk/types": "^3.664.0",
+    "@aws-sdk/types": "^3.667.0",
     "cron-parser": "^4.9.0",
     "typescript": "^5.5.4"
   },

lambdas/functions/control-plane/package.json

@@ -43,7 +43,7 @@
     "@aws-lambda-powertools/parameters": "^2.9.0",
     "@aws-sdk/client-ec2": "^3.670.0",
     "@aws-sdk/client-sqs": "^3.670.0",
-    "@aws-sdk/types": "^3.664.0",
+    "@aws-sdk/types": "^3.667.0",
     "@middy/core": "^4.7.0",
     "@octokit/auth-app": "6.1.2",
     "@octokit/core": "5.2.0",

lambdas/functions/gh-agent-syncer/package.json

@@ -39,7 +39,7 @@
     "@aws-github-runner/aws-powertools-util": "*",
     "@aws-sdk/client-s3": "^3.673.0",
     "@aws-sdk/lib-storage": "^3.673.0",
-    "@aws-sdk/types": "^3.664.0",
+    "@aws-sdk/types": "^3.667.0",
     "@middy/core": "^4.7.0",
     "@octokit/rest": "20.1.1",
     "axios": "^1.7.7"

lambdas/functions/termination-watcher/package.json

@@ -37,7 +37,7 @@
   "dependencies": {
     "@aws-github-runner/aws-powertools-util": "*",
     "@aws-sdk/client-ec2": "^3.670.0",
-    "@aws-sdk/types": "^3.664.0",
+    "@aws-sdk/types": "^3.667.0",
     "@middy/core": "^4.7.0",
     "typescript": "^5.5.4"
   },

lambdas/functions/webhook/jest.config.ts

@@ -6,10 +6,10 @@ const config: Config = {
   ...defaultConfig,
   coverageThreshold: {
     global: {
-      statements: 99.2,
+      statements: 99.58,
       branches: 100,
       functions: 100,
-      lines: 99.25,
+      lines: 99.57,
     },
   },
 };

lambdas/functions/webhook/package.json

@@ -16,6 +16,7 @@
     "all": "yarn build && yarn format && yarn lint && yarn test"
   },
   "devDependencies": {
+    "@aws-sdk/client-eventbridge": "^3.670.0",
     "@trivago/prettier-plugin-sort-imports": "^4.3.0",
     "@types/aws-lambda": "^8.10.145",
     "@types/express": "^4.17.21",

lambdas/functions/webhook/src/ConfigLoader.test.ts

@@ -0,0 +1,261 @@
+import { getParameter } from '@aws-github-runner/aws-ssm-util';
+import { ConfigWebhook, ConfigWebhookEventBridge, ConfigDispatcher } from './ConfigLoader';
+import { mocked } from 'jest-mock';
+import { logger } from '@aws-github-runner/aws-powertools-util';
+import { RunnerMatcherConfig } from './sqs';
+
+jest.mock('@aws-github-runner/aws-ssm-util');
+
+describe('ConfigLoader Tests', () => {
+  beforeEach(() => {
+    jest.clearAllMocks();
+    ConfigWebhook.reset();
+    ConfigWebhookEventBridge.reset();
+    ConfigDispatcher.reset();
+    logger.setLogLevel('DEBUG');
+
+    // clear process.env
+    for (const key of Object.keys(process.env)) {
+      delete process.env[key];
+    }
+  });
+
+  describe('Check base object', () => {
+    function setupConfiguration(): void {
+      process.env.EVENT_BUS_NAME = 'event-bus';
+      process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = '/path/to/matcher/config';
+      process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET = '/path/to/webhook/secret';
+      const matcherConfig = [
+        {
+          id: '1',
+          arn: 'arn:aws:sqs:us-east-1:123456789012:queue1',
+          fifo: false,
+          matcherConfig: {
+            labelMatchers: [['label1', 'label2']],
+            exactMatch: true,
+          },
+        },
+      ];
+      mocked(getParameter).mockImplementation(async (paramPath: string) => {
+        if (paramPath === '/path/to/matcher/config') {
+          return JSON.stringify(matcherConfig);
+        }
+        if (paramPath === '/path/to/webhook/secret') {
+          return 'secret';
+        }
+        return '';
+      });
+    }
+
+    it('should return the same instance of ConfigWebhook (singleton)', async () => {
+      setupConfiguration();
+      const config1 = await ConfigWebhook.load();
+      const config2 = await ConfigWebhook.load();
+
+      expect(config1).toBe(config2);
+      expect(getParameter).toHaveBeenCalledTimes(2);
+    });
+
+    it('should return the same instance of ConfigWebhookEventBridge (singleton)', async () => {
+      setupConfiguration();
+      const config1 = await ConfigWebhookEventBridge.load();
+      const config2 = await ConfigWebhookEventBridge.load();
+
+      expect(config1).toBe(config2);
+      expect(getParameter).toHaveBeenCalledTimes(1);
+    });
+
+    it('should return the same instance of ConfigDispatcher (singleton)', async () => {
+      setupConfiguration();
+      const config1 = await ConfigDispatcher.load();
+      const config2 = await ConfigDispatcher.load();
+
+      expect(config1).toBe(config2);
+      expect(getParameter).toHaveBeenCalledTimes(1);
+    });
+
+    it('should filter secrets from being logged', async () => {
+      setupConfiguration();
+      const spy = jest.spyOn(logger, 'debug');
+
+      await ConfigWebhook.load();
+
+      expect(spy).toHaveBeenCalledWith(
+        'Config loaded',
+        expect.objectContaining({
+          config: expect.objectContaining({
+            webhookSecret: '***',
+          }),
+        }),
+      );
+    });
+  });
+
+  describe('ConfigWebhook', () => {
+    it('should load config successfully', async () => {
+      process.env.REPOSITORY_ALLOW_LIST = '["repo1", "repo2"]';
+      process.env.SQS_WORKFLOW_JOB_QUEUE = 'secondary-queue';
+      process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = '/path/to/matcher/config';
+      process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET = '/path/to/webhook/secret';
+      const matcherConfig = [
+        {
+          id: '1',
+          arn: 'arn:aws:sqs:us-east-1:123456789012:queue1',
+          fifo: false,
+          matcherConfig: {
+            labelMatchers: [['label1', 'label2']],
+            exactMatch: true,
+          },
+        },
+      ];
+      mocked(getParameter).mockImplementation(async (paramPath: string) => {
+        if (paramPath === '/path/to/matcher/config') {
+          return JSON.stringify(matcherConfig);
+        }
+        if (paramPath === '/path/to/webhook/secret') {
+          return 'secret';
+        }
+        return '';
+      });
+
+      const config: ConfigWebhook = await ConfigWebhook.load();
+
+      expect(config.repositoryAllowList).toEqual(['repo1', 'repo2']);
+      expect(config.workflowJobEventSecondaryQueue).toBe('secondary-queue');
+      expect(config.matcherConfig).toEqual(matcherConfig);
+      expect(config.webhookSecret).toBe('secret');
+    });
+
+    it('should load config successfully', async () => {
+      process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = '/path/to/matcher/config';
+      process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET = '/path/to/webhook/secret';
+      const matcherConfig = [
+        {
+          id: '1',
+          arn: 'arn:aws:sqs:us-east-1:123456789012:queue1',
+          fifo: false,
+          matcherConfig: {
+            labelMatchers: [['label1', 'label2']],
+            exactMatch: true,
+          },
+        },
+      ];
+      mocked(getParameter).mockImplementation(async (paramPath: string) => {
+        if (paramPath === '/path/to/matcher/config') {
+          return JSON.stringify(matcherConfig);
+        }
+        if (paramPath === '/path/to/webhook/secret') {
+          return 'secret';
+        }
+        return '';
+      });
+
+      const config: ConfigWebhook = await ConfigWebhook.load();
+
+      expect(config.repositoryAllowList).toEqual([]);
+      expect(config.workflowJobEventSecondaryQueue).toBe('');
+      expect(config.matcherConfig).toEqual(matcherConfig);
+      expect(config.webhookSecret).toBe('secret');
+    });
+
+    it('should throw error if config loading fails', async () => {
+      process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = '/path/to/matcher/config';
+
+      mocked(getParameter).mockImplementation(async (paramPath: string) => {
+        if (paramPath === '/path/to/matcher/config') {
+          throw new Error('Failed to load matcher config');
+        }
+        return '';
+      });
+
+      await expect(ConfigWebhook.load()).rejects.toThrow(
+        'Failed to load config: Failed to load parameter for matcherConfig from path /path/to/matcher/config: Failed to load matcher config', // eslint-disable-line max-len
+      );
+    });
+  });
+
+  describe('ConfigWebhookEventBridge', () => {
+    it('should load config successfully', async () => {
+      process.env.ALLOWED_EVENTS = '["push", "pull_request"]';
+      process.env.EVENT_BUS_NAME = 'event-bus';
+      process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET = '/path/to/webhook/secret';
+
+      mocked(getParameter).mockImplementation(async (paramPath: string) => {
+        if (paramPath === '/path/to/webhook/secret') {
+          return 'secret';
+        }
+        return '';
+      });
+
+      const config: ConfigWebhookEventBridge = await ConfigWebhookEventBridge.load();
+
+      expect(config.allowedEvents).toEqual(['push', 'pull_request']);
+      expect(config.eventBusName).toBe('event-bus');
+      expect(config.webhookSecret).toBe('secret');
+    });
+
+    it('should throw error if config loading fails', async () => {
+      mocked(getParameter).mockImplementation(async (paramPath: string) => {
+        throw new Error(`Parameter ${paramPath} not found`);
+      });
+
+      await expect(ConfigWebhookEventBridge.load()).rejects.toThrow(
+        'Failed to load config: Environment variable for eventBusName is not set and no default value provided., Failed to load parameter for webhookSecret from path undefined: Parameter undefined not found', // eslint-disable-line max-len
+      );
+    });
+  });
+
+  describe('ConfigDispatcher', () => {
+    it('should load config successfully', async () => {
+      process.env.REPOSITORY_ALLOW_LIST = '["repo1", "repo2"]';
+      process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = '/path/to/matcher/config';
+
+      const matcherConfig: RunnerMatcherConfig[] = [
+        {
+          arn: 'arn:aws:sqs:eu-central-1:123456:npalm-default-queued-builds',
+          fifo: true,
+          id: 'https://sqs.eu-central-1.amazonaws.com/123456/npalm-default-queued-builds',
+          matcherConfig: {
+            exactMatch: true,
+            labelMatchers: [['default', 'example', 'linux', 'self-hosted', 'x64']],
+          },
+        },
+      ];
+      mocked(getParameter).mockImplementation(async (paramPath: string) => {
+        if (paramPath === '/path/to/matcher/config') {
+          return JSON.stringify(matcherConfig);
+        }
+        return '';
+      });
+
+      const config: ConfigDispatcher = await ConfigDispatcher.load();
+
+      expect(config.repositoryAllowList).toEqual(['repo1', 'repo2']);
+      expect(config.matcherConfig).toEqual(matcherConfig);
+    });
+
+    it('should throw error if config loading fails', async () => {
+      mocked(getParameter).mockImplementation(async (paramPath: string) => {
+        throw new Error(`Parameter ${paramPath} not found`);
+      });
+
+      await expect(ConfigDispatcher.load()).rejects.toThrow(
+        'Failed to load config: Failed to load parameter for matcherConfig from path undefined: Parameter undefined not found', // eslint-disable-line max-len
+      );
+    });
+
+    it('should throw an error if runner matcher config is empty.', async () => {
+      process.env.REPOSITORY_ALLOW_LIST = '["repo1", "repo2"]';
+      process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = '/path/to/matcher/config';
+
+      mocked(getParameter).mockImplementation(async (paramPath: string) => {
+        if (paramPath === '/path/to/matcher/config') {
+          return JSON.stringify('');
+        }
+        return '';
+      });
+
+      await expect(ConfigDispatcher.load()).rejects.toThrow('ailed to load config: Matcher config is empty');
+    });
+  });
+});