From 102b0e4c62426a178916d254a443ce76fdbf1efa Mon Sep 17 00:00:00 2001 From: Niek Palm Date: Sat, 9 May 2020 19:01:32 +0200 Subject: [PATCH 1/3] Create runners --- .../lambdas/scale-runners/src/local.ts | 8 +++ .../src/scale-runners/runners.ts | 54 ++++++++++++++++++- modules/runners/templates/user-data.sh | 4 +- 3 files changed, 63 insertions(+), 3 deletions(-) create mode 100644 modules/runners/lambdas/scale-runners/src/local.ts diff --git a/modules/runners/lambdas/scale-runners/src/local.ts b/modules/runners/lambdas/scale-runners/src/local.ts new file mode 100644 index 0000000000..2171f4b4a6 --- /dev/null +++ b/modules/runners/lambdas/scale-runners/src/local.ts @@ -0,0 +1,8 @@ +import { creatRunner } from './scale-runners/runners'; + +creatRunner({ + runnerConfig: '--url https://github.com/npalm/self-hosted-cowsay --token abc --label niek', + repoName: 'npalm/self-hosted-cowsay', +}).catch((e) => { + console.log(e); +}); diff --git a/modules/runners/lambdas/scale-runners/src/scale-runners/runners.ts b/modules/runners/lambdas/scale-runners/src/scale-runners/runners.ts index 0414d8f05d..a114093bcd 100644 --- a/modules/runners/lambdas/scale-runners/src/scale-runners/runners.ts +++ b/modules/runners/lambdas/scale-runners/src/scale-runners/runners.ts @@ -1,4 +1,4 @@ -import { EC2 } from 'aws-sdk'; +import { EC2, SSM } from 'aws-sdk'; export interface RunnerInfo { instanceId: string; @@ -44,3 +44,55 @@ export async function listRunners(filters: ListRunnerFilters | undefined = undef } return runners; } + +export interface RunnerInputParameters { + runnerConfig: string; + repoName?: string; + orgName?: string; +} + +export async function creatRunner(runnerParameters: RunnerInputParameters): Promise { + const launchTemplateName = process.env.LAUNCH_TEMPLATE_NAME as string; + const launchTemplateVersion = process.env.LAUNCH_TEMPLATE_VERSION as string; + const environment = process.env.ENVIRONMENT as string; + + const subnets = (process.env.SUBNET_IDS as string).split(','); + const randomSubnet = subnets[Math.floor(Math.random() * (subnets.length + 1))]; + const ec2 = new EC2(); + console.log(runnerParameters); + console.log(runnerParameters.orgName ? 'Org' : 'Repo'); + const runInstancesResponse = await ec2 + .runInstances({ + MaxCount: 1, + MinCount: 1, + LaunchTemplate: { + LaunchTemplateName: launchTemplateName, + Version: launchTemplateVersion, + }, + SubnetId: randomSubnet, + TagSpecifications: [ + { + ResourceType: 'instance', + Tags: [ + { Key: 'Application', Value: 'github-action-runner' }, + { + Key: runnerParameters.orgName ? 'Org' : 'Repo', + Value: runnerParameters.orgName ? runnerParameters.orgName : runnerParameters.repoName, + }, + ], + }, + ], + }) + .promise(); + + const ssm = new SSM(); + runInstancesResponse.Instances?.forEach((i: EC2.Instance) => { + ssm + .putParameter({ + Name: (environment + '-' + i.InstanceId) as string, + Value: runnerParameters.runnerConfig, + Type: 'SecureString', + }) + .promise(); + }); +} diff --git a/modules/runners/templates/user-data.sh b/modules/runners/templates/user-data.sh index 3d390fb174..e570ef7c9a 100644 --- a/modules/runners/templates/user-data.sh +++ b/modules/runners/templates/user-data.sh @@ -28,10 +28,10 @@ usermod -a -G docker ec2-user yum install -y curl jq git cd /home/ec2-user -mkdir actions-runner && cd actions-runner +mkdir -p actions-runner && cd actions-runner aws s3 cp ${s3_location_runner_distribution} actions-runner.tar.gz tar xzf ./actions-runner.tar.gz -rm actions-runner.tar.gz +rm -rf actions-runner.tar.gz INSTANCE_ID=$(wget -q -O - http://169.254.169.254/latest/meta-data/instance-id) REGION=$(curl -s 169.254.169.254/latest/dynamic/instance-identity/document | jq -r .region) From bc7242e811f04d8e63d62039124b9e42d984d524 Mon Sep 17 00:00:00 2001 From: Niek Palm Date: Sun, 10 May 2020 11:44:50 +0200 Subject: [PATCH 2/3] Fix user_data, refactor --- .../lambdas/scale-runners/src/local.ts | 7 ++++--- .../src/scale-runners/runners.ts | 20 ++++++++++++++----- modules/runners/main.tf | 7 +++++-- modules/runners/templates/user-data.sh | 17 ++-------------- modules/runners/variables.tf | 3 ++- 5 files changed, 28 insertions(+), 26 deletions(-) diff --git a/modules/runners/lambdas/scale-runners/src/local.ts b/modules/runners/lambdas/scale-runners/src/local.ts index 2171f4b4a6..abbe532bb8 100644 --- a/modules/runners/lambdas/scale-runners/src/local.ts +++ b/modules/runners/lambdas/scale-runners/src/local.ts @@ -1,8 +1,9 @@ -import { creatRunner } from './scale-runners/runners'; +import { createRunner } from './scale-runners/runners'; -creatRunner({ - runnerConfig: '--url https://github.com/npalm/self-hosted-cowsay --token abc --label niek', +createRunner({ + runnerConfig: '--url https://github.com/npalm/self-hosted-cowsay --token abc', repoName: 'npalm/self-hosted-cowsay', + environment: 'default', }).catch((e) => { console.log(e); }); diff --git a/modules/runners/lambdas/scale-runners/src/scale-runners/runners.ts b/modules/runners/lambdas/scale-runners/src/scale-runners/runners.ts index a114093bcd..32da62b913 100644 --- a/modules/runners/lambdas/scale-runners/src/scale-runners/runners.ts +++ b/modules/runners/lambdas/scale-runners/src/scale-runners/runners.ts @@ -46,21 +46,21 @@ export async function listRunners(filters: ListRunnerFilters | undefined = undef } export interface RunnerInputParameters { + runnerName: string; runnerConfig: string; + environment: string; repoName?: string; orgName?: string; } -export async function creatRunner(runnerParameters: RunnerInputParameters): Promise { +async function createEC2Runner(runnerParameters: RunnerInputParameters): Promise { const launchTemplateName = process.env.LAUNCH_TEMPLATE_NAME as string; const launchTemplateVersion = process.env.LAUNCH_TEMPLATE_VERSION as string; - const environment = process.env.ENVIRONMENT as string; const subnets = (process.env.SUBNET_IDS as string).split(','); const randomSubnet = subnets[Math.floor(Math.random() * (subnets.length + 1))]; + const ec2 = new EC2(); - console.log(runnerParameters); - console.log(runnerParameters.orgName ? 'Org' : 'Repo'); const runInstancesResponse = await ec2 .runInstances({ MaxCount: 1, @@ -89,10 +89,20 @@ export async function creatRunner(runnerParameters: RunnerInputParameters): Prom runInstancesResponse.Instances?.forEach((i: EC2.Instance) => { ssm .putParameter({ - Name: (environment + '-' + i.InstanceId) as string, + Name: runnerParameters.environment + '-' + (i.InstanceId as string), Value: runnerParameters.runnerConfig, Type: 'SecureString', }) .promise(); }); } + +export async function createRunner(runnerParameters: RunnerInputParameters): Promise { + const runners = await listRunners({ repoName: runnerParameters.repoName, orgName: runnerParameters.orgName }); + + if (runners?.length < 1) { + await createEC2Runner(runnerParameters); + } else { + console.info('No runner will be created, maximum number of runners reached.'); + } +} diff --git a/modules/runners/main.tf b/modules/runners/main.tf index 3431ffa763..ba268b0fda 100644 --- a/modules/runners/main.tf +++ b/modules/runners/main.tf @@ -1,15 +1,18 @@ locals { - name_sg = var.overrides["name_sg"] == "" ? local.tags["Name"] : var.overrides["name_sg"] tags = merge( { - "Name" = format("%s", var.environment) + "Name" = format("%s-action-runner", var.environment) }, { "Environment" = format("%s", var.environment) }, var.tags, ) + + name_sg = var.overrides["name_sg"] == "" ? local.tags["Name"] : var.overrides["name_sg"] + name_runner = var.overrides["name_runner"] == "" ? local.tags["Name"] : var.overrides["name_runner"] + } data "aws_ami" "runner" { diff --git a/modules/runners/templates/user-data.sh b/modules/runners/templates/user-data.sh index e570ef7c9a..e54d54c131 100644 --- a/modules/runners/templates/user-data.sh +++ b/modules/runners/templates/user-data.sh @@ -8,27 +8,14 @@ amazon-linux-extras install docker service docker start usermod -a -G docker ec2-user -# Install runner yum install -y curl jq git -cd /home/ec2-user -mkdir actions-runner && cd actions-runner -#!/bin/bash -ex -exec > >(tee /var/log/user-data.log | logger -t user-data -s 2>/dev/console) 2>&1 - -yum update -y ${pre_install} -# Install docker -amazon-linux-extras install docker -service docker start -usermod -a -G docker ec2-user - # Install runner -yum install -y curl jq git - cd /home/ec2-user -mkdir -p actions-runner && cd actions-runner +mkdir actions-runner && cd actions-runner + aws s3 cp ${s3_location_runner_distribution} actions-runner.tar.gz tar xzf ./actions-runner.tar.gz rm -rf actions-runner.tar.gz diff --git a/modules/runners/variables.tf b/modules/runners/variables.tf index e95d387964..adc391d3c7 100644 --- a/modules/runners/variables.tf +++ b/modules/runners/variables.tf @@ -13,7 +13,8 @@ variable "overrides" { type = map(string) default = { - name_sg = "" + name_runner = "" + name_sg = "" } } From 6c0b6dcc82ca40c5b2b42ceb01a3fc160fe54dfb Mon Sep 17 00:00:00 2001 From: Niek Palm Date: Sun, 10 May 2020 14:50:52 +0200 Subject: [PATCH 3/3] Connect create instance to lambda, update policies, minor fixes --- examples/default/main.tf | 15 ++++++------ examples/default/outputs.tf | 7 +----- examples/default/variables.tf | 4 ---- main.tf | 1 + .../lambdas/scale-runners/src/local.ts | 9 ------- .../src/scale-runners/handler.ts | 17 ++++++++++--- .../src/scale-runners/runners.ts | 24 ++++++++----------- modules/runners/policies/lambda-scale-up.json | 5 ++++ modules/runners/scale-runners-lambda.tf | 5 ++++ modules/runners/variables.tf | 5 ++++ variables.tf | 6 +++++ 11 files changed, 54 insertions(+), 44 deletions(-) delete mode 100644 modules/runners/lambdas/scale-runners/src/local.ts diff --git a/examples/default/main.tf b/examples/default/main.tf index cfe77e4aed..a2b763d2eb 100644 --- a/examples/default/main.tf +++ b/examples/default/main.tf @@ -8,26 +8,25 @@ resource "random_password" "random" { length = 32 } - module "runners" { source = "../../" aws_region = local.aws_region vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets environment = local.environment tags = { Project = "ProjectX" } - github_app_webhook_secret = var.github_app_webhook_secret - - github_app_client_id = var.github_app_client_id - github_app_client_secret = var.github_app_client_secret - github_app_id = var.github_app_id - github_app_key_base64 = var.github_app_key_base64 + github_app_client_id = var.github_app_client_id + github_app_client_secret = var.github_app_client_secret + github_app_id = var.github_app_id + github_app_key_base64 = var.github_app_key_base64 + github_app_webhook_secret = random_password.random.result - enable_organization_runners = var.enable_organization_runners + enable_organization_runners = false } diff --git a/examples/default/outputs.tf b/examples/default/outputs.tf index 1d66ed1d49..1d2d3a8cee 100644 --- a/examples/default/outputs.tf +++ b/examples/default/outputs.tf @@ -4,14 +4,9 @@ output "runners" { } } -# output "binaries_syncer" { -# value = { -# binaries_syncer = module.runners.binaries_syncer -# } -# } - output "webhook" { value = { + secret = random_password.random.result gateway = module.runners.webhook.gateway } } diff --git a/examples/default/variables.tf b/examples/default/variables.tf index c45e02f469..5701717dcf 100644 --- a/examples/default/variables.tf +++ b/examples/default/variables.tf @@ -1,6 +1,3 @@ -variable "enable_organization_runners" { - type = bool -} variable "github_app_key_base64" {} @@ -10,4 +7,3 @@ variable "github_app_client_id" {} variable "github_app_client_secret" {} -variable "github_app_webhook_secret" {} diff --git a/main.tf b/main.tf index d233b71f9c..43960095f0 100644 --- a/main.tf +++ b/main.tf @@ -38,6 +38,7 @@ module "runners" { aws_region = var.aws_region vpc_id = var.vpc_id + subnet_ids = var.subnet_ids environment = var.environment tags = local.tags diff --git a/modules/runners/lambdas/scale-runners/src/local.ts b/modules/runners/lambdas/scale-runners/src/local.ts deleted file mode 100644 index abbe532bb8..0000000000 --- a/modules/runners/lambdas/scale-runners/src/local.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { createRunner } from './scale-runners/runners'; - -createRunner({ - runnerConfig: '--url https://github.com/npalm/self-hosted-cowsay --token abc', - repoName: 'npalm/self-hosted-cowsay', - environment: 'default', -}).catch((e) => { - console.log(e); -}); diff --git a/modules/runners/lambdas/scale-runners/src/scale-runners/handler.ts b/modules/runners/lambdas/scale-runners/src/scale-runners/handler.ts index d3dee407eb..556201e66b 100644 --- a/modules/runners/lambdas/scale-runners/src/scale-runners/handler.ts +++ b/modules/runners/lambdas/scale-runners/src/scale-runners/handler.ts @@ -1,7 +1,7 @@ import { createAppAuth } from '@octokit/auth-app'; import { Octokit } from '@octokit/rest'; import { AppAuth } from '@octokit/auth-app/dist-types/types'; -import { listRunners } from './runners'; +import { listRunners, createRunner } from './runners'; import yn from 'yn'; export interface ActionRequestMessage { @@ -34,8 +34,9 @@ async function createInstallationClient(githubAppAuth: AppAuth): Promise => { if (eventSource !== 'aws:sqs') throw Error('Cannot handle non-SQS events!'); - const enableOrgLevel = yn(process.env.ENABLE_ORGANIZATION_RUNNERS); + const enableOrgLevel = yn(process.env.ENABLE_ORGANIZATION_RUNNERS, { default: true }); const maximumRunners = parseInt(process.env.RUNNERS_MAXIMUM_COUNT || '3'); + const environment = process.env.ENVIRONMENT as string; const githubAppAuth = createGithubAppAuth(payload.installationId); const githubInstallationClient = await createInstallationClient(githubAppAuth); const queuedWorkflows = await githubInstallationClient.actions.listRepoWorkflowRuns({ @@ -70,7 +71,17 @@ export const handle = async (eventSource: string, payload: ActionRequestMessage) repo: payload.repositoryName, }); const token = registrationToken.data.token; - // create runner + + await createRunner({ + environment: environment, + runnerConfig: enableOrgLevel + ? `--url https://github.com/${payload.repositoryOwner} --token ${token}` + : `--url https://github.com/${payload.repositoryOwner}/${payload.repositoryName} --token ${token}`, + orgName: enableOrgLevel ? payload.repositoryOwner : undefined, + repoName: enableOrgLevel ? undefined : `${payload.repositoryOwner}/${payload.repositoryName}`, + }); + } else { + console.info('No runner will be created, maximum number of runners reached.'); } } }; diff --git a/modules/runners/lambdas/scale-runners/src/scale-runners/runners.ts b/modules/runners/lambdas/scale-runners/src/scale-runners/runners.ts index 32da62b913..711cd6a488 100644 --- a/modules/runners/lambdas/scale-runners/src/scale-runners/runners.ts +++ b/modules/runners/lambdas/scale-runners/src/scale-runners/runners.ts @@ -46,19 +46,18 @@ export async function listRunners(filters: ListRunnerFilters | undefined = undef } export interface RunnerInputParameters { - runnerName: string; runnerConfig: string; environment: string; repoName?: string; orgName?: string; } -async function createEC2Runner(runnerParameters: RunnerInputParameters): Promise { +export async function createRunner(runnerParameters: RunnerInputParameters): Promise { const launchTemplateName = process.env.LAUNCH_TEMPLATE_NAME as string; const launchTemplateVersion = process.env.LAUNCH_TEMPLATE_VERSION as string; const subnets = (process.env.SUBNET_IDS as string).split(','); - const randomSubnet = subnets[Math.floor(Math.random() * (subnets.length + 1))]; + const randomSubnet = subnets[Math.floor(Math.random() * subnets.length)]; const ec2 = new EC2(); const runInstancesResponse = await ec2 @@ -84,25 +83,22 @@ async function createEC2Runner(runnerParameters: RunnerInputParameters): Promise ], }) .promise(); + console.info( + 'Created instance(s): ', + runInstancesResponse.Instances?.forEach((i: EC2.Instance) => { + i.InstanceId; + }), + ); const ssm = new SSM(); runInstancesResponse.Instances?.forEach((i: EC2.Instance) => { - ssm + const r = ssm .putParameter({ Name: runnerParameters.environment + '-' + (i.InstanceId as string), Value: runnerParameters.runnerConfig, Type: 'SecureString', }) .promise(); + console.log(r); }); } - -export async function createRunner(runnerParameters: RunnerInputParameters): Promise { - const runners = await listRunners({ repoName: runnerParameters.repoName, orgName: runnerParameters.orgName }); - - if (runners?.length < 1) { - await createEC2Runner(runnerParameters); - } else { - console.info('No runner will be created, maximum number of runners reached.'); - } -} diff --git a/modules/runners/policies/lambda-scale-up.json b/modules/runners/policies/lambda-scale-up.json index 82a9e75d8a..743a7b908a 100644 --- a/modules/runners/policies/lambda-scale-up.json +++ b/modules/runners/policies/lambda-scale-up.json @@ -15,6 +15,11 @@ "Effect": "Allow", "Action": "iam:PassRole", "Resource": "${arn_runner_instance_role}" + }, + { + "Effect": "Allow", + "Action": ["ssm:PutParameter"], + "Resource": "*" } ] } diff --git a/modules/runners/scale-runners-lambda.tf b/modules/runners/scale-runners-lambda.tf index c13e9dfa76..8ae90b6554 100644 --- a/modules/runners/scale-runners-lambda.tf +++ b/modules/runners/scale-runners-lambda.tf @@ -5,6 +5,7 @@ resource "aws_lambda_function" "scale_runners_lambda" { role = aws_iam_role.scale_runners_lambda.arn handler = "index.handler" runtime = "nodejs12.x" + timeout = 60 environment { variables = { @@ -13,6 +14,10 @@ resource "aws_lambda_function" "scale_runners_lambda" { GITHUB_APP_ID = var.github_app_id GITHUB_APP_CLIENT_ID = var.github_app_client_id GITHUB_APP_CLIENT_SECRET = var.github_app_client_secret + SUBNET_IDS = join(",", var.subnet_ids) + LAUNCH_TEMPLATE_NAME = aws_launch_template.runner.name + LAUNCH_TEMPLATE_VERSION = aws_launch_template.runner.latest_version + ENVIRONMENT = var.environment } } } diff --git a/modules/runners/variables.tf b/modules/runners/variables.tf index adc391d3c7..cb357e1476 100644 --- a/modules/runners/variables.tf +++ b/modules/runners/variables.tf @@ -8,6 +8,11 @@ variable "vpc_id" { type = string } +variable "subnet_ids" { + description = "List of subnets in which the action runners will be launched, the subnets needs to be subnets in the `vpc_id`." + type = list(string) +} + variable "overrides" { description = "This maps provides the possibility to override some defaults. The following attributes are supported: `name_sg` overwrite the `Name` tag for all security groups created by this module. `name_runner_agent_instance` override the `Name` tag for the ec2 instance defined in the auto launch configuration. `name_docker_machine_runners` ovverrid the `Name` tag spot instances created by the runner agent." type = map(string) diff --git a/variables.tf b/variables.tf index 2c7ec146d4..c51c069731 100644 --- a/variables.tf +++ b/variables.tf @@ -8,6 +8,12 @@ variable "vpc_id" { type = string } +variable "subnet_ids" { + description = "List of subnets in which the action runners will be launched, the subnets needs to be subnets in the `vpc_id`." + type = list(string) +} + + variable "tags" { description = "Map of tags that will be added to created resources. By default resources will be tagged with name and environment." type = map(string)