feat: added changes to enable tracing in lambdas.

README.md

@@ -540,6 +540,7 @@ We welcome any improvement to the standard module to make the default as secure
 | <a name="input_lambda_s3_bucket"></a> [lambda\_s3\_bucket](#input\_lambda\_s3\_bucket) | S3 bucket from which to specify lambda functions. This is an alternative to providing local files directly. | `string` | `null` | no |
 | <a name="input_lambda_security_group_ids"></a> [lambda\_security\_group\_ids](#input\_lambda\_security\_group\_ids) | List of security group IDs associated with the Lambda function. | `list(string)` | `[]` | no |
 | <a name="input_lambda_subnet_ids"></a> [lambda\_subnet\_ids](#input\_lambda\_subnet\_ids) | List of subnets in which the action runners will be launched, the subnets needs to be subnets in the `vpc_id`. | `list(string)` | `[]` | no |
+| <a name="input_lambda_tracing_config"></a> [lambda\_tracing\_config](#input\_lambda\_tracing\_config) | Configuration for lambda tracing. | <pre>object({<br>    capture_http_requests = optional(bool, false)<br>    capture_error         = optional(bool, false)<br>  })</pre> | `null` | no |
 | <a name="input_lambda_tracing_mode"></a> [lambda\_tracing\_mode](#input\_lambda\_tracing\_mode) | Enable X-Ray tracing for the lambda functions. | `string` | `null` | no |
 | <a name="input_log_level"></a> [log\_level](#input\_log\_level) | Logging level for lambda logging. Valid values are  'silly', 'trace', 'debug', 'info', 'warn', 'error', 'fatal'. | `string` | `"info"` | no |
 | <a name="input_log_type"></a> [log\_type](#input\_log\_type) | Logging format for lambda logging. Valid values are 'json', 'pretty', 'hidden'. | `string` | `null` | no |

lambdas/functions/control-plane/package.json

@@ -40,6 +40,7 @@
   "dependencies": {
     "@aws-sdk/client-ec2": "^3.350.0",
     "@aws-sdk/types": "^3.347.0",
+    "@middy/core": "^3.6.2",
     "@octokit/auth-app": "6.0.1",
     "@octokit/rest": "20.0.2",
     "@octokit/types": "^12.0.0",

lambdas/functions/control-plane/src/aws/runners.ts

@@ -8,6 +8,7 @@ import {
   TerminateInstancesCommand,
 } from '@aws-sdk/client-ec2';
 import { createChildLogger } from '@terraform-aws-github-runner/aws-powertools-util';
+import { getTracedAWSV3Client } from '@terraform-aws-github-runner/aws-powertools-util';
 import { getParameter } from '@terraform-aws-github-runner/aws-ssm-util';
 import moment from 'moment';
 
@@ -55,7 +56,7 @@ function constructFilters(filters?: Runners.ListRunnerFilters): Ec2Filter[][] {
 }
 
 async function getRunners(ec2Filters: Ec2Filter[]): Promise<Runners.RunnerList[]> {
-  const ec2 = new EC2Client({ region: process.env.AWS_REGION });
+  const ec2 = getTracedAWSV3Client<EC2Client>(new EC2Client({ region: process.env.AWS_REGION }));
   const runners: Runners.RunnerList[] = [];
   let nextToken;
   let hasNext = true;

lambdas/functions/control-plane/src/lambda.ts

@@ -1,4 +1,6 @@
+import middy from '@middy/core';
 import { logger, setContext } from '@terraform-aws-github-runner/aws-powertools-util';
+import { captureLambdaHandler, tracer } from '@terraform-aws-github-runner/aws-powertools-util';
 import { Context, SQSEvent } from 'aws-lambda';
 import 'source-map-support/register';
 
@@ -7,6 +9,9 @@ import ScaleError from './scale-runners/ScaleError';
 import { scaleDown } from './scale-runners/scale-down';
 import { scaleUp } from './scale-runners/scale-up';
 
+export const handlerScaleUp = middy(scaleUpHandler).use(captureLambdaHandler(tracer));
+export const handlerScaleDown = middy(scaleDownHandler).use(captureLambdaHandler(tracer));
+export const handlerPool = middy(adjustPool).use(captureLambdaHandler(tracer));
 export async function scaleUpHandler(event: SQSEvent, context: Context): Promise<void> {
   setContext(context, 'lambda.ts');
   logger.logEventIfEnabled(event);

lambdas/functions/gh-agent-syncer/package.json

@@ -40,6 +40,7 @@
     "@aws-sdk/client-s3": "^3.350.0",
     "@aws-sdk/lib-storage": "^3.350.0",
     "@aws-sdk/types": "^3.347.0",
+    "@middy/core": "^3.6.2",
     "@terraform-aws-github-runner/aws-powertools-util": "*",
     "axios": "^1.3.5"
   }

lambdas/functions/gh-agent-syncer/src/lambda.ts

@@ -1,8 +1,11 @@
+import middy from '@middy/core';
 import { logger, setContext } from '@terraform-aws-github-runner/aws-powertools-util';
+import { captureLambdaHandler, tracer } from '@terraform-aws-github-runner/aws-powertools-util';
 import { Context } from 'aws-lambda';
 
 import { sync } from './syncer/syncer';
 
+export const lambdaHandler = middy(handler).use(captureLambdaHandler(tracer));
 // eslint-disable-next-line
 export async function handler(event: any, context: Context): Promise<void> {
   setContext(context, 'lambda.ts');

lambdas/functions/gh-agent-syncer/src/syncer/syncer.ts

@@ -2,6 +2,7 @@ import { GetObjectTaggingCommand, S3Client, Tag } from '@aws-sdk/client-s3';
 import { Upload } from '@aws-sdk/lib-storage';
 import { Octokit } from '@octokit/rest';
 import { createChildLogger } from '@terraform-aws-github-runner/aws-powertools-util';
+import { getTracedAWSV3Client } from '@terraform-aws-github-runner/aws-powertools-util';
 import axios from 'axios';
 import { Stream } from 'stream';
 
@@ -84,7 +85,7 @@ async function uploadToS3(
 }
 
 export async function sync(): Promise<void> {
-  const s3 = new S3Client({});
+  const s3 = getTracedAWSV3Client<S3Client>(new S3Client({}));
 
   const runnerOs = process.env.GITHUB_RUNNER_OS || 'linux';
   const runnerArch = process.env.GITHUB_RUNNER_ARCHITECTURE || 'x64';

lambdas/functions/webhook/package.json

@@ -39,6 +39,7 @@
   },
   "dependencies": {
     "@aws-sdk/client-sqs": "^3.315.0",
+    "@middy/core": "^3.6.2",
     "@octokit/rest": "^20.0.1",
     "@octokit/types": "^12.0.0",
     "@octokit/webhooks": "^12.0.3",

lambdas/functions/webhook/src/lambda.ts

@@ -1,4 +1,6 @@
+import middy from '@middy/core';
 import { logger, setContext } from '@terraform-aws-github-runner/aws-powertools-util';
+import { captureLambdaHandler, tracer } from '@terraform-aws-github-runner/aws-powertools-util';
 import { APIGatewayEvent, Context } from 'aws-lambda';
 
 import { handle } from './webhook/handler';
@@ -7,6 +9,7 @@ export interface Response {
   statusCode: number;
   body?: string;
 }
+export const handlerWebhook = middy(githubWebhook).use(captureLambdaHandler(tracer));
 export async function githubWebhook(event: APIGatewayEvent, context: Context): Promise<Response> {
   setContext(context, 'lambda.ts');
   logger.logEventIfEnabled(event);

lambdas/functions/webhook/src/sqs/index.ts

@@ -1,6 +1,7 @@
 import { SQS, SendMessageCommandInput } from '@aws-sdk/client-sqs';
 import { WorkflowJobEvent } from '@octokit/webhooks-types';
 import { createChildLogger } from '@terraform-aws-github-runner/aws-powertools-util';
+import { getTracedAWSV3Client } from '@terraform-aws-github-runner/aws-powertools-util';
 
 const logger = createChildLogger('sqs');
 
@@ -30,7 +31,7 @@ export interface GithubWorkflowEvent {
 }
 
 export const sendActionRequest = async (message: ActionRequestMessage): Promise<void> => {
-  const sqs = new SQS({ region: process.env.AWS_REGION });
+  const sqs = getTracedAWSV3Client<SQS>(new SQS({ region: process.env.AWS_REGION }));
 
   const sqsMessage: SendMessageCommandInput = {
     QueueUrl: message.queueId,

lambdas/libs/aws-powertools-util/package.json

@@ -37,6 +37,7 @@
   },
   "dependencies": {
     "@aws-lambda-powertools/logger": "^1.8.0",
+    "@aws-lambda-powertools/tracer": "^1.14.0",
     "aws-lambda": "^1.0.7"
   }
 }

lambdas/libs/aws-powertools-util/src/index.ts

@@ -1 +1,2 @@
 export * from './logger';
+export * from './tracer';

lambdas/libs/aws-powertools-util/src/tracer/index.ts

@@ -0,0 +1,10 @@
+import { Tracer, captureLambdaHandler } from '@aws-lambda-powertools/tracer';
+
+const tracer = new Tracer({
+  serviceName: process.env.SERVICE_NAME || 'runners',
+});
+
+function getTracedAWSV3Client<T>(client: T): T {
+  return tracer.captureAWSClient(client);
+}
+export { tracer, captureLambdaHandler, getTracedAWSV3Client };

main.tf

@@ -155,6 +155,7 @@ module "webhook" {
   lambda_zip                                    = var.webhook_lambda_zip
   lambda_timeout                                = var.webhook_lambda_timeout
   lambda_tracing_mode                           = var.lambda_tracing_mode
+  lambda_tracing_config                         = var.lambda_tracing_config
   logging_retention_in_days                     = var.logging_retention_in_days
   logging_kms_key_id                            = var.logging_kms_key_id
 
@@ -239,6 +240,7 @@ module "runners" {
   lambda_subnet_ids                = var.lambda_subnet_ids
   lambda_security_group_ids        = var.lambda_security_group_ids
   lambda_tracing_mode              = var.lambda_tracing_mode
+  lambda_tracing_config            = var.lambda_tracing_config
   logging_retention_in_days        = var.logging_retention_in_days
   logging_kms_key_id               = var.logging_kms_key_id
   enable_cloudwatch_agent          = var.enable_cloudwatch_agent
@@ -302,6 +304,7 @@ module "runner_binaries" {
   lambda_zip                      = var.runner_binaries_syncer_lambda_zip
   lambda_timeout                  = var.runner_binaries_syncer_lambda_timeout
   lambda_tracing_mode             = var.lambda_tracing_mode
+  lambda_tracing_config           = var.lambda_tracing_config
   logging_retention_in_days       = var.logging_retention_in_days
   logging_kms_key_id              = var.logging_kms_key_id
 

modules/multi-runner/README.md

@@ -134,6 +134,7 @@ module "multi-runner" {
 | <a name="input_lambda_s3_bucket"></a> [lambda\_s3\_bucket](#input\_lambda\_s3\_bucket) | S3 bucket from which to specify lambda functions. This is an alternative to providing local files directly. | `string` | `null` | no |
 | <a name="input_lambda_security_group_ids"></a> [lambda\_security\_group\_ids](#input\_lambda\_security\_group\_ids) | List of security group IDs associated with the Lambda function. | `list(string)` | `[]` | no |
 | <a name="input_lambda_subnet_ids"></a> [lambda\_subnet\_ids](#input\_lambda\_subnet\_ids) | List of subnets in which the action runners will be launched, the subnets needs to be subnets in the `vpc_id`. | `list(string)` | `[]` | no |
+| <a name="input_lambda_tracing_config"></a> [lambda\_tracing\_config](#input\_lambda\_tracing\_config) | Configuration for lambda tracing. | <pre>object({<br>    capture_http_requests = optional(bool, false)<br>    capture_error         = optional(bool, false)<br>  })</pre> | `null` | no |
 | <a name="input_lambda_tracing_mode"></a> [lambda\_tracing\_mode](#input\_lambda\_tracing\_mode) | Enable X-Ray tracing for the lambda functions. | `string` | `null` | no |
 | <a name="input_log_level"></a> [log\_level](#input\_log\_level) | Logging level for lambda logging. Valid values are  'silly', 'trace', 'debug', 'info', 'warn', 'error', 'fatal'. | `string` | `"info"` | no |
 | <a name="input_log_type"></a> [log\_type](#input\_log\_type) | Logging format for lambda logging. Valid values are 'json', 'pretty', 'hidden'. | `string` | `null` | no |

modules/multi-runner/runner-binaries.tf

@@ -18,6 +18,7 @@ module "runner_binaries" {
   lambda_zip                        = var.runner_binaries_syncer_lambda_zip
   lambda_timeout                    = var.runner_binaries_syncer_lambda_timeout
   lambda_tracing_mode               = var.lambda_tracing_mode
+  lambda_tracing_config             = var.lambda_tracing_config
   logging_retention_in_days         = var.logging_retention_in_days
   logging_kms_key_id                = var.logging_kms_key_id
   enable_event_rule_binaries_syncer = var.enable_event_rule_binaries_syncer

modules/multi-runner/runners.tf

@@ -67,6 +67,7 @@ module "runners" {
   lambda_subnet_ids                = var.lambda_subnet_ids
   lambda_security_group_ids        = var.lambda_security_group_ids
   lambda_tracing_mode              = var.lambda_tracing_mode
+  lambda_tracing_config            = var.lambda_tracing_config
   logging_retention_in_days        = var.logging_retention_in_days
   logging_kms_key_id               = var.logging_kms_key_id
   enable_cloudwatch_agent          = each.value.runner_config.enable_cloudwatch_agent

modules/multi-runner/variables.tf

@@ -556,3 +556,12 @@ variable "lambda_tracing_mode" {
   type        = string
   default     = null
 }
+
+variable "lambda_tracing_config" {
+  description = "Configuration for lambda tracing."
+  type = object({
+    capture_http_requests = optional(bool, false)
+    capture_error         = optional(bool, false)
+  })
+  default = null
+}

modules/multi-runner/webhook.tf

@@ -20,6 +20,7 @@ module "webhook" {
   lambda_zip                                    = var.webhook_lambda_zip
   lambda_timeout                                = var.webhook_lambda_timeout
   lambda_tracing_mode                           = var.lambda_tracing_mode
+  lambda_tracing_config                         = var.lambda_tracing_config
   logging_retention_in_days                     = var.logging_retention_in_days
   logging_kms_key_id                            = var.logging_kms_key_id
 

modules/runner-binaries-syncer/README.md

@@ -100,6 +100,7 @@ No modules.
 | <a name="input_lambda_security_group_ids"></a> [lambda\_security\_group\_ids](#input\_lambda\_security\_group\_ids) | List of security group IDs associated with the Lambda function. | `list(string)` | `[]` | no |
 | <a name="input_lambda_subnet_ids"></a> [lambda\_subnet\_ids](#input\_lambda\_subnet\_ids) | List of subnets in which the action runners will be launched, the subnets needs to be subnets in the `vpc_id`. | `list(string)` | `[]` | no |
 | <a name="input_lambda_timeout"></a> [lambda\_timeout](#input\_lambda\_timeout) | Time out of the lambda in seconds. | `number` | `300` | no |
+| <a name="input_lambda_tracing_config"></a> [lambda\_tracing\_config](#input\_lambda\_tracing\_config) | Configuration for lambda tracing. | <pre>object({<br>    capture_http_requests = optional(bool, false)<br>    capture_error         = optional(bool, false)<br>  })</pre> | `null` | no |
 | <a name="input_lambda_tracing_mode"></a> [lambda\_tracing\_mode](#input\_lambda\_tracing\_mode) | Enable X-Ray tracing for the lambda functions. | `string` | `null` | no |
 | <a name="input_lambda_zip"></a> [lambda\_zip](#input\_lambda\_zip) | File location of the lambda zip file. | `string` | `null` | no |
 | <a name="input_log_level"></a> [log\_level](#input\_log\_level) | Logging level for lambda logging. Valid values are  'silly', 'trace', 'debug', 'info', 'warn', 'error', 'fatal'. | `string` | `"info"` | no |

modules/runner-binaries-syncer/runner-binaries-syncer.tf

@@ -23,15 +23,18 @@ resource "aws_lambda_function" "syncer" {
 
   environment {
     variables = {
-      ENVIRONMENT                 = var.prefix
-      GITHUB_RUNNER_ARCHITECTURE  = var.runner_architecture
-      GITHUB_RUNNER_OS            = local.gh_binary_os_label[var.runner_os]
-      LOG_LEVEL                   = var.log_level
-      POWERTOOLS_LOGGER_LOG_EVENT = var.log_level == "debug" ? "true" : "false"
-      S3_BUCKET_NAME              = aws_s3_bucket.action_dist.id
-      S3_OBJECT_KEY               = local.action_runner_distribution_object_key
-      S3_SSE_ALGORITHM            = try(var.server_side_encryption_configuration.rule.apply_server_side_encryption_by_default.sse_algorithm, null)
-      S3_SSE_KMS_KEY_ID           = try(var.server_side_encryption_configuration.rule.apply_server_side_encryption_by_default.kms_master_key_id, null)
+      ENVIRONMENT                              = var.prefix
+      GITHUB_RUNNER_ARCHITECTURE               = var.runner_architecture
+      GITHUB_RUNNER_OS                         = local.gh_binary_os_label[var.runner_os]
+      LOG_LEVEL                                = var.log_level
+      POWERTOOLS_LOGGER_LOG_EVENT              = var.log_level == "debug" ? "true" : "false"
+      POWERTOOLS_TRACE_ENABLED                 = var.lambda_tracing_mode == "Active" ? true : false
+      POWERTOOLS_TRACER_CAPTURE_HTTPS_REQUESTS = var.lambda_tracing_config.capture_http_requests
+      POWERTOOLS_TRACER_CAPTURE_ERROR          = var.lambda_tracing_config.capture_error
+      S3_BUCKET_NAME                           = aws_s3_bucket.action_dist.id
+      S3_OBJECT_KEY                            = local.action_runner_distribution_object_key
+      S3_SSE_ALGORITHM                         = try(var.server_side_encryption_configuration.rule.apply_server_side_encryption_by_default.sse_algorithm, null)
+      S3_SSE_KMS_KEY_ID                        = try(var.server_side_encryption_configuration.rule.apply_server_side_encryption_by_default.kms_master_key_id, null)
     }
   }
 

modules/runner-binaries-syncer/variables.tf

@@ -253,3 +253,12 @@ variable "lambda_tracing_mode" {
   type        = string
   default     = null
 }
+
+variable "lambda_tracing_config" {
+  description = "Configuration for lambda tracing."
+  type = object({
+    capture_http_requests = optional(bool, false)
+    capture_error         = optional(bool, false)
+  })
+  default = null
+}

modules/runners/README.md

@@ -74,7 +74,6 @@ yarn run dist
 | Name | Type |
 |------|------|
 | [aws_cloudwatch_event_rule.scale_down](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_rule) | resource |
-| [aws_cloudwatch_event_target.scale_down](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_target) | resource |
 | [aws_cloudwatch_log_group.gh_runners](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource |
 | [aws_cloudwatch_log_group.scale_down](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource |
 | [aws_cloudwatch_log_group.scale_up](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource |
@@ -100,6 +99,7 @@ yarn run dist
 | [aws_iam_role_policy_attachment.managed_policies](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_iam_role_policy_attachment.scale_down_vpc_execution_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_iam_role_policy_attachment.scale_up_vpc_execution_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
+| [aws_iam_role_policy_attachment.xray_tracing](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_lambda_event_source_mapping.scale_up](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_event_source_mapping) | resource |
 | [aws_lambda_function.scale_down](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_function) | resource |
 | [aws_lambda_function.scale_up](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_function) | resource |
@@ -166,6 +166,7 @@ yarn run dist
 | <a name="input_lambda_subnet_ids"></a> [lambda\_subnet\_ids](#input\_lambda\_subnet\_ids) | List of subnets in which the lambda will be launched, the subnets needs to be subnets in the `vpc_id`. | `list(string)` | `[]` | no |
 | <a name="input_lambda_timeout_scale_down"></a> [lambda\_timeout\_scale\_down](#input\_lambda\_timeout\_scale\_down) | Time out for the scale down lambda in seconds. | `number` | `60` | no |
 | <a name="input_lambda_timeout_scale_up"></a> [lambda\_timeout\_scale\_up](#input\_lambda\_timeout\_scale\_up) | Time out for the scale up lambda in seconds. | `number` | `60` | no |
+| <a name="input_lambda_tracing_config"></a> [lambda\_tracing\_config](#input\_lambda\_tracing\_config) | Configuration for lambda tracing. | <pre>object({<br>    capture_http_requests = optional(bool, false)<br>    capture_error         = optional(bool, false)<br>  })</pre> | `null` | no |
 | <a name="input_lambda_tracing_mode"></a> [lambda\_tracing\_mode](#input\_lambda\_tracing\_mode) | Enable X-Ray tracing for the lambda functions. | `string` | `null` | no |
 | <a name="input_lambda_zip"></a> [lambda\_zip](#input\_lambda\_zip) | File location of the lambda zip file. | `string` | `null` | no |
 | <a name="input_log_level"></a> [log\_level](#input\_log\_level) | Logging level for lambda logging. Valid values are  'silly', 'trace', 'debug', 'info', 'warn', 'error', 'fatal'. | `string` | `"info"` | no |

modules/runners/policies-runner.tf

@@ -44,6 +44,11 @@ resource "aws_iam_role_policy" "dist_bucket" {
   )
 }
 
+resource "aws_iam_role_policy_attachment" "xray_tracing" {
+  role       = aws_iam_role.runner.name
+  policy_arn = "arn:aws:iam::aws:policy/AWSXRayDaemonWriteAccess"
+}
+
 resource "aws_iam_role_policy" "describe_tags" {
   name   = "runner-describe-tags"
   role   = aws_iam_role.runner.name

modules/runners/policies/instance-xray-policy.json

@@ -0,0 +1,16 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Sid": "AllowXRay",
+      "Effect": "Allow",
+      "Action": [
+        "xray:BatchGetTraces",
+        "xray:GetTraceSummaries",
+        "xray:PutTelemetryRecords",
+        "xray:PutTraceSegments"
+      ],
+      "Resource": ["*"]
+    }
+  ]
+}