Feature/runners lambdas wip npalm (#9)

* Create runners

* Fix user_data, refactor

* Connect create instance to lambda, update policies, minor fixes

Author: npalm

examples/default/main.tf

@@ -8,26 +8,25 @@ resource "random_password" "random" {
   length = 32
 }
 
-
 module "runners" {
   source = "../../"
 
   aws_region = local.aws_region
   vpc_id     = module.vpc.vpc_id
+  subnet_ids = module.vpc.private_subnets
 
   environment = local.environment
   tags = {
     Project = "ProjectX"
   }
 
-  github_app_webhook_secret = var.github_app_webhook_secret
-
-  github_app_client_id     = var.github_app_client_id
-  github_app_client_secret = var.github_app_client_secret
-  github_app_id            = var.github_app_id
-  github_app_key_base64    = var.github_app_key_base64
+  github_app_client_id      = var.github_app_client_id
+  github_app_client_secret  = var.github_app_client_secret
+  github_app_id             = var.github_app_id
+  github_app_key_base64     = var.github_app_key_base64
+  github_app_webhook_secret = random_password.random.result
 
-  enable_organization_runners = var.enable_organization_runners
+  enable_organization_runners = false
 }
 
 

examples/default/outputs.tf

@@ -4,14 +4,9 @@ output "runners" {
   }
 }
 
-# output "binaries_syncer" {
-#   value = {
-#     binaries_syncer = module.runners.binaries_syncer
-#   }
-# }
-
 output "webhook" {
   value = {
+    secret  = random_password.random.result
     gateway = module.runners.webhook.gateway
   }
 }

examples/default/variables.tf

@@ -1,6 +1,3 @@
-variable "enable_organization_runners" {
-  type = bool
-}
 
 variable "github_app_key_base64" {}
 
@@ -10,4 +7,3 @@ variable "github_app_client_id" {}
 
 variable "github_app_client_secret" {}
 
-variable "github_app_webhook_secret" {}

main.tf

@@ -38,6 +38,7 @@ module "runners" {
 
   aws_region  = var.aws_region
   vpc_id      = var.vpc_id
+  subnet_ids  = var.subnet_ids
   environment = var.environment
   tags        = local.tags
 

modules/runners/lambdas/scale-runners/src/scale-runners/handler.ts

@@ -1,7 +1,7 @@
 import { createAppAuth } from '@octokit/auth-app';
 import { Octokit } from '@octokit/rest';
 import { AppAuth } from '@octokit/auth-app/dist-types/types';
-import { listRunners } from './runners';
+import { listRunners, createRunner } from './runners';
 import yn from 'yn';
 
 export interface ActionRequestMessage {
@@ -34,8 +34,9 @@ async function createInstallationClient(githubAppAuth: AppAuth): Promise<Octokit
 
 export const handle = async (eventSource: string, payload: ActionRequestMessage): Promise<void> => {
   if (eventSource !== 'aws:sqs') throw Error('Cannot handle non-SQS events!');
-  const enableOrgLevel = yn(process.env.ENABLE_ORGANIZATION_RUNNERS);
+  const enableOrgLevel = yn(process.env.ENABLE_ORGANIZATION_RUNNERS, { default: true });
   const maximumRunners = parseInt(process.env.RUNNERS_MAXIMUM_COUNT || '3');
+  const environment = process.env.ENVIRONMENT as string;
   const githubAppAuth = createGithubAppAuth(payload.installationId);
   const githubInstallationClient = await createInstallationClient(githubAppAuth);
   const queuedWorkflows = await githubInstallationClient.actions.listRepoWorkflowRuns({
@@ -70,7 +71,17 @@ export const handle = async (eventSource: string, payload: ActionRequestMessage)
             repo: payload.repositoryName,
           });
       const token = registrationToken.data.token;
-      // create runner
+
+      await createRunner({
+        environment: environment,
+        runnerConfig: enableOrgLevel
+          ? `--url https://github.com/${payload.repositoryOwner} --token ${token}`
+          : `--url https://github.com/${payload.repositoryOwner}/${payload.repositoryName} --token ${token}`,
+        orgName: enableOrgLevel ? payload.repositoryOwner : undefined,
+        repoName: enableOrgLevel ? undefined : `${payload.repositoryOwner}/${payload.repositoryName}`,
+      });
+    } else {
+      console.info('No runner will be created, maximum number of runners reached.');
     }
   }
 };

modules/runners/lambdas/scale-runners/src/scale-runners/runners.ts

@@ -1,4 +1,4 @@
-import { EC2 } from 'aws-sdk';
+import { EC2, SSM } from 'aws-sdk';
 
 export interface RunnerInfo {
   instanceId: string;
@@ -44,3 +44,61 @@ export async function listRunners(filters: ListRunnerFilters | undefined = undef
   }
   return runners;
 }
+
+export interface RunnerInputParameters {
+  runnerConfig: string;
+  environment: string;
+  repoName?: string;
+  orgName?: string;
+}
+
+export async function createRunner(runnerParameters: RunnerInputParameters): Promise<void> {
+  const launchTemplateName = process.env.LAUNCH_TEMPLATE_NAME as string;
+  const launchTemplateVersion = process.env.LAUNCH_TEMPLATE_VERSION as string;
+
+  const subnets = (process.env.SUBNET_IDS as string).split(',');
+  const randomSubnet = subnets[Math.floor(Math.random() * subnets.length)];
+
+  const ec2 = new EC2();
+  const runInstancesResponse = await ec2
+    .runInstances({
+      MaxCount: 1,
+      MinCount: 1,
+      LaunchTemplate: {
+        LaunchTemplateName: launchTemplateName,
+        Version: launchTemplateVersion,
+      },
+      SubnetId: randomSubnet,
+      TagSpecifications: [
+        {
+          ResourceType: 'instance',
+          Tags: [
+            { Key: 'Application', Value: 'github-action-runner' },
+            {
+              Key: runnerParameters.orgName ? 'Org' : 'Repo',
+              Value: runnerParameters.orgName ? runnerParameters.orgName : runnerParameters.repoName,
+            },
+          ],
+        },
+      ],
+    })
+    .promise();
+  console.info(
+    'Created instance(s): ',
+    runInstancesResponse.Instances?.forEach((i: EC2.Instance) => {
+      i.InstanceId;
+    }),
+  );
+
+  const ssm = new SSM();
+  runInstancesResponse.Instances?.forEach((i: EC2.Instance) => {
+    const r = ssm
+      .putParameter({
+        Name: runnerParameters.environment + '-' + (i.InstanceId as string),
+        Value: runnerParameters.runnerConfig,
+        Type: 'SecureString',
+      })
+      .promise();
+    console.log(r);
+  });
+}

modules/runners/main.tf

@@ -1,15 +1,18 @@
 locals {
-  name_sg = var.overrides["name_sg"] == "" ? local.tags["Name"] : var.overrides["name_sg"]
 
   tags = merge(
     {
-      "Name" = format("%s", var.environment)
+      "Name" = format("%s-action-runner", var.environment)
     },
     {
       "Environment" = format("%s", var.environment)
     },
     var.tags,
   )
+
+  name_sg     = var.overrides["name_sg"] == "" ? local.tags["Name"] : var.overrides["name_sg"]
+  name_runner = var.overrides["name_runner"] == "" ? local.tags["Name"] : var.overrides["name_runner"]
+
 }
 
 data "aws_ami" "runner" {

modules/runners/policies/lambda-scale-up.json

@@ -15,6 +15,11 @@
       "Effect": "Allow",
       "Action": "iam:PassRole",
       "Resource": "${arn_runner_instance_role}"
+    },
+    {
+      "Effect": "Allow",
+      "Action": ["ssm:PutParameter"],
+      "Resource": "*"
     }
   ]
 }

modules/runners/scale-runners-lambda.tf

@@ -5,6 +5,7 @@ resource "aws_lambda_function" "scale_runners_lambda" {
   role             = aws_iam_role.scale_runners_lambda.arn
   handler          = "index.handler"
   runtime          = "nodejs12.x"
+  timeout          = 60
 
   environment {
     variables = {
@@ -13,6 +14,10 @@ resource "aws_lambda_function" "scale_runners_lambda" {
       GITHUB_APP_ID               = var.github_app_id
       GITHUB_APP_CLIENT_ID        = var.github_app_client_id
       GITHUB_APP_CLIENT_SECRET    = var.github_app_client_secret
+      SUBNET_IDS                  = join(",", var.subnet_ids)
+      LAUNCH_TEMPLATE_NAME        = aws_launch_template.runner.name
+      LAUNCH_TEMPLATE_VERSION     = aws_launch_template.runner.latest_version
+      ENVIRONMENT                 = var.environment
     }
   }
 }

modules/runners/templates/user-data.sh

@@ -8,30 +8,17 @@ amazon-linux-extras install docker
 service docker start
 usermod -a -G docker ec2-user
 
-# Install runner
 yum install -y curl jq git
-cd /home/ec2-user
-mkdir actions-runner && cd actions-runner
-#!/bin/bash -ex
-exec > >(tee /var/log/user-data.log | logger -t user-data -s 2>/dev/console) 2>&1
-
-yum update -y
 
 ${pre_install}
 
-# Install docker
-amazon-linux-extras install docker
-service docker start
-usermod -a -G docker ec2-user
-
 # Install runner
-yum install -y curl jq git
-
 cd /home/ec2-user
 mkdir actions-runner && cd actions-runner
+
 aws s3 cp ${s3_location_runner_distribution} actions-runner.tar.gz
 tar xzf ./actions-runner.tar.gz
-rm actions-runner.tar.gz
+rm -rf actions-runner.tar.gz
 
 INSTANCE_ID=$(wget -q -O - http://169.254.169.254/latest/meta-data/instance-id)
 REGION=$(curl -s 169.254.169.254/latest/dynamic/instance-identity/document | jq -r .region)

modules/runners/variables.tf

@@ -8,12 +8,18 @@ variable "vpc_id" {
   type        = string
 }
 
+variable "subnet_ids" {
+  description = "List of subnets in which the action runners will be launched, the subnets needs to be subnets in the `vpc_id`."
+  type        = list(string)
+}
+
 variable "overrides" {
   description = "This maps provides the possibility to override some defaults. The following attributes are supported: `name_sg` overwrite the `Name` tag for all security groups created by this module. `name_runner_agent_instance` override the `Name` tag for the ec2 instance defined in the auto launch configuration. `name_docker_machine_runners` ovverrid the `Name` tag spot instances created by the runner agent."
   type        = map(string)
 
   default = {
-    name_sg = ""
+    name_runner = ""
+    name_sg     = ""
   }
 }
 

variables.tf

@@ -8,6 +8,12 @@ variable "vpc_id" {
   type        = string
 }
 
+variable "subnet_ids" {
+  description = "List of subnets in which the action runners will be launched, the subnets needs to be subnets in the `vpc_id`."
+  type        = list(string)
+}
+
+
 variable "tags" {
   description = "Map of tags that will be added to created resources. By default resources will be tagged with name and environment."
   type        = map(string)