feat: Add spot termination handler

Author: npalm

docs/configuration.md

@@ -215,21 +215,35 @@ In case the setup does not work as intended, trace the events through this seque
 
 ### Termination watcher
 
-This feature is in early stage and therefore disabled by default.
+This feature is in early stage and therefore disabled by default. To enable the watcher set `instance_termination_watcher.enable = true`.
 
-The termination watcher is currently watching for spot termination notifications. The module is only taken events into account for instances tagged with `ghr:environment` by default when deployment the module as part of one of the main modules (root or multi-runner). The module can also be deployed stand-alone, in that case the tag filter needs to be tunned.
+The termination watcher is currently watching for spot terminations. The module is only taken events into account for instances tagged with `ghr:environment` by default when deployment the module as part of one of the main modules (root or multi-runner). The module can also be deployed stand-alone, in that case the tag filter needs to be tunned.
+
+### Termination notification
+
+The watcher is listening for spot termination warnings and create a log message and optionally a metric. The watcher is disabled by default. The feature is enabled once the watcher is enabled, the feature can be disabled explicit by setting `instance_termination_watcher.features.enable_spot_termination_handler = false`.
 
 - Logs: The module will log all termination notifications. For each warning it will look up instance details and log the environment, instance type and time the instance is running. As well some other details.
 - Metrics: Metrics are disabled by default, this to avoid costs. Once enabled a metric will be created for each warning with at least dimensions for the environment and instance type. THe metric name space can be configured via the variables. The metric name used is `SpotInterruptionWarning`.
 
-#### Log example
+### Termination handler
+
+!!! warning
+    This feature will only work once the CloudTrail is enabled.
+
+The termination handler is listening for spot terminations by capture the `BidEvictedEvent` via CloudTrail. The handler will log and optionally create a metric for each termination. The intend is to enhance the logic to inform the user about the termination via the GitHub Job or Workflow run. The feature is disabled by default. The feature is enabled once the watcher is enabled, the feature can be disabled explicit by setting `instance_termination_watcher.features.enable_spot_termination_handler = false`.
+
+- Logs: The module will log all termination notifications. For each warning it will look up instance details and log the environment, instance type and time the instance is running. As well some other details.
+- Metrics: Metrics are disabled by default, this to avoid costs. Once enabled a metric will be created for each termination with at least dimensions for the environment and instance type. THe metric name space can be configured via the variables. The metric name used is `SpotTermination`.
+
+### Log example (both warnings and terminations)
 
 Below an example of the the log messages created.
 
 ```
 {
     "level": "INFO",
-    "message": "Received spot notification warning:",
+    "message": "Received spot notification for ${metricName}",
     "environment": "default",
     "instanceId": "i-0039b8826b3dcea55",
     "instanceType": "c5.large",

examples/default/main.tf

@@ -78,7 +78,7 @@ module "runners" {
   # Let the module manage the service linked role
   # create_service_linked_role_spot = true
 
-  instance_types = ["m5.large", "c5.large"]
+  instance_types = ["m7a.large", "m5.large"]
 
   # override delay of events in seconds
   delay_webhook_event   = 5
@@ -98,7 +98,7 @@ module "runners" {
   runner_name_prefix = "${local.environment}_"
 
   # Enable debug logging for the lambda functions
-  log_level = "info"
+  log_level = "debug"
 
   enable_ami_housekeeper = true
   ami_housekeeper_cleanup_config = {
@@ -117,14 +117,14 @@ module "runners" {
   }
 
   # enable metric creation  (experimental)
-  # metrics = {
-  #   enable = true
-  #   metric = {
-  #     enable_spot_termination_warning = true
-  #     enable_job_retry                = false
-  #     enable_github_app_rate_limit    = true
-  #   }
-  # }
+  metrics = {
+    enable = true
+    metric = {
+      enable_spot_termination_warning = true
+      enable_job_retry                = false
+      enable_github_app_rate_limit    = false
+    }
+  }
 
   # enable job_retry feature. Be careful with this feature, it can lead to you hitting API rate limits.
   # job_retry = {

lambdas/functions/termination-watcher/src/ConfigResolver.ts

@@ -2,6 +2,7 @@ import { createChildLogger } from '@aws-github-runner/aws-powertools-util';
 
 export class Config {
   createSpotWarningMetric: boolean;
+  createSpotTerminationMetric: boolean;
   tagFilters: Record<string, string>;
   prefix: string;
 
@@ -11,6 +12,7 @@ export class Config {
     logger.debug('Loading config from environment variables', { env: process.env });
 
     this.createSpotWarningMetric = process.env.ENABLE_METRICS_SPOT_WARNING === 'true';
+    this.createSpotTerminationMetric = process.env.ENABLE_METRICS_SPOT_TERMINATION === 'true';
     this.prefix = process.env.PREFIX ?? '';
     this.tagFilters = { 'ghr:environment': this.prefix };
 

lambdas/functions/termination-watcher/src/lambda.test.ts

@@ -3,14 +3,16 @@ import { Context } from 'aws-lambda';
 import { mocked } from 'jest-mock';
 
 import { handle as interruptionWarningHandlerImpl } from './termination-warning';
-import { interruptionWarning } from './lambda';
-import { SpotInterruptionWarning, SpotTerminationDetail } from './types';
+import { handle as terminationHandlerImpl } from './termination';
+import { interruptionWarning, termination } from './lambda';
+import { BidEvictedDetail, BidEvictedEvent, SpotInterruptionWarning, SpotTerminationDetail } from './types';
 
 jest.mock('./termination-warning');
+jest.mock('./termination');
 
 process.env.POWERTOOLS_METRICS_NAMESPACE = 'test';
 process.env.POWERTOOLS_TRACE_ENABLED = 'true';
-const event: SpotInterruptionWarning<SpotTerminationDetail> = {
+const spotInstanceInterruptionEvent: SpotInterruptionWarning<SpotTerminationDetail> = {
   version: '0',
   id: '1',
   'detail-type': 'EC2 Spot Instance Interruption Warning',
@@ -25,6 +27,42 @@ const event: SpotInterruptionWarning<SpotTerminationDetail> = {
   },
 };
 
+const bidEvictedEvent: BidEvictedEvent<BidEvictedDetail> = {
+  version: '0',
+  id: '186d7999-3121-e749-23f3-c7caec1084e1',
+  'detail-type': 'AWS Service Event via CloudTrail',
+  source: 'aws.ec2',
+  account: '123456789012',
+  time: '2024-10-09T11:48:46Z',
+  region: 'eu-west-1',
+  resources: [],
+  detail: {
+    eventVersion: '1.10',
+    userIdentity: {
+      accountId: '123456789012',
+      invokedBy: 'sec2.amazonaws.com',
+    },
+    eventTime: '2024-10-09T11:48:46Z',
+    eventSource: 'ec2.amazonaws.com',
+    eventName: 'BidEvictedEvent',
+    awsRegion: 'eu-west-1',
+    sourceIPAddress: 'ec2.amazonaws.com',
+    userAgent: 'ec2.amazonaws.com',
+    requestParameters: null,
+    responseElements: null,
+    requestID: 'ebf032e3-5009-3484-aae8-b4946ab2e2eb',
+    eventID: '3a15843b-96c2-41b1-aac1-7d62dc754547',
+    readOnly: false,
+    eventType: 'AwsServiceEvent',
+    managementEvent: true,
+    recipientAccountId: '123456789012',
+    serviceEventDetails: {
+      instanceIdSet: ['i-12345678901234567'],
+    },
+    eventCategory: 'Management',
+  },
+};
+
 const context: Context = {
   awsRequestId: '1',
   callbackWaitsForEmptyEventLoop: false,
@@ -48,22 +86,52 @@ const context: Context = {
 
 // Docs for testing async with jest: https://jestjs.io/docs/tutorial-async
 describe('Handle sport termination interruption warning', () => {
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+
   it('should not throw or log in error.', async () => {
     const mock = mocked(interruptionWarningHandlerImpl);
     mock.mockImplementation(() => {
       return new Promise((resolve) => {
         resolve();
       });
     });
-    await expect(interruptionWarning(event, context)).resolves.not.toThrow();
+    await expect(interruptionWarning(spotInstanceInterruptionEvent, context)).resolves.not.toThrow();
   });
 
   it('should not throw only log in error in case of an exception.', async () => {
     const logSpy = jest.spyOn(logger, 'error');
     const error = new Error('An error.');
     const mock = mocked(interruptionWarningHandlerImpl);
     mock.mockRejectedValue(error);
-    await expect(interruptionWarning(event, context)).resolves.toBeUndefined();
+    await expect(interruptionWarning(spotInstanceInterruptionEvent, context)).resolves.toBeUndefined();
+
+    expect(logSpy).toHaveBeenCalledTimes(1);
+  });
+});
+
+describe('Handle sport termination (BidEvictEvent', () => {
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+
+  it('should not throw or log in error.', async () => {
+    const mock = mocked(terminationHandlerImpl);
+    mock.mockImplementation(() => {
+      return new Promise((resolve) => {
+        resolve();
+      });
+    });
+    await expect(termination(bidEvictedEvent, context)).resolves.not.toThrow();
+  });
+
+  it('should not throw only log in error in case of an exception.', async () => {
+    const logSpy = jest.spyOn(logger, 'error');
+    const error = new Error('An error.');
+    const mock = mocked(terminationHandlerImpl);
+    mock.mockRejectedValue(error);
+    await expect(termination(bidEvictedEvent, context)).resolves.toBeUndefined();
 
     expect(logSpy).toHaveBeenCalledTimes(1);
   });

lambdas/functions/termination-watcher/src/lambda.ts

@@ -4,7 +4,8 @@ import { logMetrics } from '@aws-lambda-powertools/metrics/middleware';
 import { Context } from 'aws-lambda';
 
 import { handle as handleTerminationWarning } from './termination-warning';
-import { SpotInterruptionWarning, SpotTerminationDetail } from './types';
+import { handle as handleTermination } from './termination';
+import { BidEvictedDetail, BidEvictedEvent, SpotInterruptionWarning, SpotTerminationDetail } from './types';
 import { Config } from './ConfigResolver';
 
 const config = new Config();
@@ -24,6 +25,18 @@ export async function interruptionWarning(
   }
 }
 
+export async function termination(event: BidEvictedEvent<BidEvictedDetail>, context: Context): Promise<void> {
+  setContext(context, 'lambda.ts');
+  logger.logEventIfEnabled(event);
+  logger.debug('Configuration of the lambda', { config });
+
+  try {
+    await handleTermination(event, config);
+  } catch (e) {
+    logger.error(`${(e as Error).message}`, { error: e as Error });
+  }
+}
+
 const addMiddleware = () => {
   const middleware = middy(interruptionWarning);