refactor: structure of lambda code (#3224)

Author: npalm

Diff for: .ci/Dockerfile

@@ -1,26 +1,15 @@
 #syntax=docker/dockerfile:1.2
-FROM node:16 as build
-WORKDIR /lambda
+FROM node:18 as build
+WORKDIR /lambdas
 RUN apt-get update \
         && apt-get install -y zip \
         && rm -rf /var/lib/apt/lists/*
 
-FROM build as runner-binaries-syncer
-COPY modules/runner-binaries-syncer/lambdas/runner-binaries-syncer /lambda
-RUN --mount=type=cache,target=/lambda/node_modules,id=runner-binaries-syncer \
-        yarn install && yarn dist
-
-FROM build as runners
-COPY modules/runners/lambdas/runners /lambda
-RUN --mount=type=cache,target=/lambda/node_modules,id=runners \
-        yarn install && yarn dist
-
-FROM build as webhook
-COPY modules/webhook/lambdas/webhook /lambda
-RUN --mount=type=cache,target=/lambda/node_modules,id=webhook \
-        yarn install && yarn dist
+FROM build as lambdas
+COPY lambdas /lambdas
+RUN --mount=type=cache,target=/work/node_modules,id=lambdas \
+        yarn install --frozen-lockfile && yarn dist && \
+        find . -name "*.zip" -exec cp {} . \;
 
 FROM scratch as final
-COPY --from=runner-binaries-syncer /lambda/runner-binaries-syncer.zip /runner-binaries-syncer.zip
-COPY --from=runners                /lambda/runners.zip                /runners.zip
-COPY --from=webhook                /lambda/webhook.zip                /webhook.zip
+COPY --from=lambdas /lambdas/*.zip /

Diff for: .ci/build-yarn.sh

@@ -1,10 +1,10 @@
-name: Lambda Syncer
+name: Build lambdas
 on:
-  workflow_call:
-    inputs:
-      working-directory:
-        required: true
-        type: string
+  pull_request:
+    branches:
+      - main
+    paths:
+      - 'lambdas/**'
 
 jobs:
   build:
@@ -16,12 +16,12 @@ jobs:
       image: node:${{ matrix.node }}
     defaults:
       run:
-        working-directory: ${{ inputs.working-directory }}/${{ inputs.image }}
+        working-directory: ./lambdas
 
     steps:
       - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # v3.2.0
       - name: Install dependencies
-        run: yarn install
+        run: yarn install --frozen-lockfile
       - name: Run prettier
         run: yarn format-check
       - name: Run linter

Diff for: .github/workflows/lambda-runner-binaries-syncer.yml

@@ -5,16 +5,14 @@ on:
       - main
       - v1
   workflow_dispatch:
+
 jobs:
-  prepare:
-    name: Create dist
+  release:
+    name: Release
     runs-on: ubuntu-latest
     container: node:18
-    strategy:
-      matrix:
-        lambda: ["modules/webhook/lambdas/webhook", "modules/runner-binaries-syncer/lambdas/runner-binaries-syncer", "modules/runners/lambdas/runners"]
     permissions:
-      contents: read
+      contents: write
       actions: write
     steps:
       - name: Extract lambda name
@@ -26,21 +24,8 @@ jobs:
       - name: Add zip
         run: apt update && apt install zip
       - name: Build dist
-        working-directory: ${{ matrix.lambda }}
+        working-directory: lambdas
         run: yarn install && yarn run test && yarn dist
-      - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # ratchet:actions/upload-artifact@v3
-        with:
-          name: ${{ steps.lambda.outputs.name }}
-          path: ${{ matrix.lambda }}/${{ steps.lambda.outputs.name }}.zip
-          retention-days: 1
-  release:
-    name: release
-    runs-on: ubuntu-latest
-    needs: prepare
-    outputs:
-      releases_created: ${{ steps.release.outputs.releases_created }}
-      tag_name: ${{ steps.release.outputs.tag_name }}
-    steps:
       - name: Get installation token
         uses: philips-software/app-token-action@a37926571e4cec6f219e06727136efdd073d8657 # ratchet:philips-software/[email protected]
         id: token
@@ -59,23 +44,11 @@ jobs:
           default-branch: ${{ steps.branch.outputs.name }}
           release-type: terraform-module
           token: ${{ steps.token.outputs.token }}
-  assets:
-    name: upload assets
-    if: ${{ needs.release.outputs.releases_created }}
-    needs: release
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        asset: ["webhook", "runner-binaries-syncer", "runners"]
-    permissions:
-      contents: write
-      actions: read
-    steps:
-      - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # ratchet:actions/checkout@v3
-      - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # ratchet:actions/download-artifact@v3
-        with:
-          name: ${{ matrix.asset }}
       - name: Upload Release Asset
+        if: ${{ steps.release.outputs.releases_created }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: gh release upload ${{ needs.release.outputs.tag_name }} ${{ matrix.asset }}.zip
+        run: |
+          for f in $(find . -name '*.zip'); do
+            gh release upload ${{ needs.release.outputs.tag_name }} $f
+          done

Diff for: .github/workflows/lambda-runners.yml

@@ -27,9 +27,9 @@ jobs:
         uses: actions/checkout@v3
       - name: "Fake zip files" # Validate will fail if it cannot find the zip files
         run: |
-          touch modules/webhook/lambdas/webhook/webhook.zip
-          touch modules/runners/lambdas/runners/runners.zip
-          touch modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/runner-binaries-syncer.zip
+          touch lambdas/functions/webhook/webhook.zip
+          touch lambdas/functions/control-plane/runners.zip
+          touch lambdas/functions/gh-agent-syncer/runner-binaries-syncer.zip
       - name: terraform init
         run: terraform init -get -backend=false -input=false
       - if: contains(matrix.terraform, '1.3.')

Diff for: .github/workflows/lambda-webhook.yml

@@ -1,7 +1 @@
-{
-    "sonarlint.rules": {
-        "javascript:S4123": {
-            "level": "off"
-        }
-    }
-}
+{}

Diff for: .github/workflows/lambda-template.yml renamed to .github/workflows/lambda.yml

@@ -0,0 +1,8 @@
+node_modules/
+build/
+dist/
+*.log
+
+# Ignore all yarn.lock files except the one in the root
+**/yarn.lock
+!/yarn.lock

Diff for: .github/workflows/release.yml

@@ -0,0 +1,17 @@
+import type { Config } from 'jest';
+
+import defaultConfig from '../../jest.base.config';
+
+const config: Config = {
+  ...defaultConfig,
+  coverageThreshold: {
+    global: {
+      statements: 97,
+      branches: 93,
+      functions: 96,
+      lines: 98,
+    },
+  },
+};
+
+export default config;

Diff for: .github/workflows/terraform.yml

@@ -1,5 +1,5 @@
 {
-  "name": "github-runner-lambda-scale-runners",
+  "name": "@terraform-aws-github-runner/control-plane",
   "version": "1.0.0",
   "main": "lambda.ts",
   "license": "MIT",
@@ -16,10 +16,8 @@
     "all": "yarn build && yarn format && yarn lint && yarn test"
   },
   "devDependencies": {
-    "@babel/helper-get-function-arity": "^7.16.7",
     "@trivago/prettier-plugin-sort-imports": "^4.1.1",
     "@types/aws-lambda": "^8.10.114",
-    "@types/express": "^4.17.17",
     "@types/jest": "^29.5.0",
     "@types/node": "^18.11.18",
     "@typescript-eslint/eslint-plugin": "^5.57.0",
@@ -34,19 +32,19 @@
     "jest-mock-extended": "^3.0.4",
     "moment-timezone": "^0.5.43",
     "nock": "^13.3.0",
-    "prettier": "2.8.7",
+    "prettier": "2.8.8",
     "ts-jest": "^29.1.0",
     "ts-node": "^10.9.1",
     "ts-node-dev": "^2.0.0"
   },
   "dependencies": {
-    "@aws-lambda-powertools/logger": "^1.8.0",
     "@aws-sdk/client-ec2": "^3.303.0",
-    "@aws-sdk/client-ssm": "^3.303.0",
     "@octokit/auth-app": "4.0.9",
     "@octokit/rest": "^19.0.7",
     "@octokit/types": "^9.0.0",
+    "@terraform-aws-github-runner/aws-ssm-util": "*",
+    "@terraform-aws-github-runner/aws-powertools-util": "*",
     "cron-parser": "^4.8.1",
-    "typescript": "^4.9.5"
+    "typescript": "^5.0.4"
   }
 }

Diff for: .vscode/settings.json

@@ -8,9 +8,9 @@ import {
   TerminateInstancesCommand,
 } from '@aws-sdk/client-ec2';
 import { SSM } from '@aws-sdk/client-ssm';
+import { createChildLogger } from '@terraform-aws-github-runner/aws-powertools-util';
 import moment from 'moment';
 
-import { createChildLogger } from '../logger';
 import ScaleError from './../scale-runners/ScaleError';
 import * as Runners from './runners.d';
 

Diff for: modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/.eslintrc.yaml renamed to lambdas/.eslintrc.yaml

@@ -2,14 +2,14 @@ import { createAppAuth } from '@octokit/auth-app';
 import { StrategyOptions } from '@octokit/auth-app/dist-types/types';
 import { request } from '@octokit/request';
 import { RequestInterface } from '@octokit/types';
+import { getParameterValue } from '@terraform-aws-github-runner/aws-ssm-util';
 import { mocked } from 'jest-mock';
 import { MockProxy, mock } from 'jest-mock-extended';
 import nock from 'nock';
 
-import { getParameterValue } from './../aws/ssm';
 import { createGithubAppAuth, createOctoClient } from './gh-auth';
 
-jest.mock('./../aws/ssm');
+jest.mock('@terraform-aws-github-runner/aws-ssm-util');
 jest.mock('@octokit/auth-app');
 
 const cleanEnv = process.env;

Diff for: lambdas/.gitignore

@@ -10,9 +10,8 @@ import {
 import { OctokitOptions } from '@octokit/core/dist-types/types';
 import { request } from '@octokit/request';
 import { Octokit } from '@octokit/rest';
-
-import { getParameterValue } from '../aws/ssm';
-import { createChildLogger } from '../logger';
+import { createChildLogger } from '@terraform-aws-github-runner/aws-powertools-util';
+import { getParameterValue } from '@terraform-aws-github-runner/aws-ssm-util';
 
 const logger = createChildLogger('gh-auth');
 

Diff for: modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/.nvmrc renamed to lambdas/.nvmrc

@@ -1,8 +1,8 @@
+import { logger } from '@terraform-aws-github-runner/aws-powertools-util';
 import { Context, SQSEvent, SQSRecord } from 'aws-lambda';
 import { mocked } from 'jest-mock';
 
 import { adjustPool, scaleDownHandler, scaleUpHandler } from './lambda';
-import { logger } from './logger';
 import { adjust } from './pool/pool';
 import ScaleError from './scale-runners/ScaleError';
 import { scaleDown } from './scale-runners/scale-down';
@@ -61,7 +61,7 @@ const context: Context = {
 jest.mock('./scale-runners/scale-up');
 jest.mock('./scale-runners/scale-down');
 jest.mock('./pool/pool');
-jest.mock('./logger');
+jest.mock('@terraform-aws-github-runner/aws-powertools-util');
 
 // Docs for testing async with jest: https://jestjs.io/docs/tutorial-async
 describe('Test scale up lambda wrapper.', () => {

Diff for: modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/.prettierrc renamed to lambdas/.prettierrc

@@ -1,7 +1,7 @@
+import { logger, setContext } from '@terraform-aws-github-runner/aws-powertools-util';
 import { Context, SQSEvent } from 'aws-lambda';
 import 'source-map-support/register';
 
-import { logger, setContext } from './logger';
 import { PoolEvent, adjust } from './pool/pool';
 import ScaleError from './scale-runners/ScaleError';
 import { scaleDown } from './scale-runners/scale-down';

Diff for: lambdas/functions/control-plane/jest.config.ts

@@ -1,4 +1,5 @@
-import { logger } from './logger';
+import { logger } from '@terraform-aws-github-runner/aws-powertools-util';
+
 import { ActionRequestMessage, scaleUp } from './scale-runners/scale-up';
 
 const sqsEvent = {

Diff for: modules/runners/lambdas/runners/package.json renamed to lambdas/functions/control-plane/package.json

@@ -1,8 +1,8 @@
+import { createChildLogger } from '@terraform-aws-github-runner/aws-powertools-util';
 import yn from 'yn';
 
 import { bootTimeExceeded, listEC2Runners } from '../aws/runners';
 import { createGithubAppAuth, createGithubInstallationAuth, createOctoClient } from '../gh-auth/gh-auth';
-import { createChildLogger } from '../logger';
 import { createRunners } from '../scale-runners/scale-up';
 
 const logger = createChildLogger('pool');

Diff for: modules/runners/lambdas/runners/src/aws/runners.d.ts renamed to lambdas/functions/control-plane/src/aws/runners.d.ts

@@ -1,8 +1,8 @@
 import { Octokit } from '@octokit/rest';
+import { createChildLogger } from '@terraform-aws-github-runner/aws-powertools-util';
 import moment from 'moment';
 
 import { createGithubAppAuth, createGithubInstallationAuth, createOctoClient } from '../gh-auth/gh-auth';
-import { createChildLogger } from '../logger';
 import { bootTimeExceeded, listEC2Runners, terminateRunner } from './../aws/runners';
 import { RunnerInfo, RunnerList } from './../aws/runners.d';
 import { GhRunners, githubCache } from './cache';

Diff for: modules/runners/lambdas/runners/src/aws/runners.test.ts renamed to lambdas/functions/control-plane/src/aws/runners.test.ts

@@ -1,8 +1,8 @@
 import { Octokit } from '@octokit/rest';
+import { addPersistentContextToChildLogger, createChildLogger } from '@terraform-aws-github-runner/aws-powertools-util';
 import yn from 'yn';
 
 import { createGithubAppAuth, createGithubInstallationAuth, createOctoClient } from '../gh-auth/gh-auth';
-import { addPersistentContextToChildLogger, createChildLogger } from '../logger';
 import { createRunner, listEC2Runners } from './../aws/runners';
 import { RunnerInputParameters } from './../aws/runners.d';
 import ScaleError from './ScaleError';