chore(e2e-tests): cleanup lambdas in initialize workflow, update run-e2e-ci script with additional logging (#1080)

Author: dphang

.github/workflows/e2e-tests.yml

@@ -87,6 +87,16 @@ jobs:
           path: build.tar.gz
           retention-days: 1
 
+      - name: Cleanup any old Lambda versions in AWS account
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_AT }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_ST }}
+          AWS_DEFAULT_REGION: us-east-1
+        run: |
+          cd packages/e2e-tests/test-utils
+          yarn --frozen-lockfile
+          yarn cleanup-lambda
+
   run-e2e-tests:
     needs: [initialize]
     runs-on: [ubuntu-latest]

packages/e2e-tests/test-utils/package.json

@@ -2,6 +2,9 @@
   "name": "e2e-test-utils",
   "version": "1.0.0",
   "description": "Test utils for running serverless-next.js end-to-end tests",
+  "scripts": {
+    "cleanup-lambda": "ts-node --project tsconfig.scripts.json scripts/cleanup-lambda.ts"
+  },
   "repository": {
     "type": "git",
     "url": "git+https://github.com/serverless-next.js/serverless-next.js"

packages/e2e-tests/test-utils/scripts/cleanup-lambda.ts

@@ -0,0 +1,137 @@
+#!/usr/bin/env node
+
+// Cleanup Lambda code adapted from https://github.com/davidmenger/cleanup-lambda-versions/blob/master/src/cleanupVersions.js
+import * as AWS from "aws-sdk";
+import {
+  ListFunctionsResponse,
+  ListVersionsByFunctionResponse
+} from "aws-sdk/clients/lambda";
+
+function listLambdaFunctions(
+  lambda: AWS.Lambda,
+  nextMarker: string | undefined = undefined
+): Promise<ListFunctionsResponse> {
+  return new Promise((resolve, reject) => {
+    const opts = {
+      MaxItems: 999
+    };
+
+    if (nextMarker) {
+      Object.assign(opts, { Marker: nextMarker });
+    }
+
+    lambda.listFunctions(opts, (err, res) => {
+      if (err) {
+        reject(err);
+      } else {
+        resolve(res);
+      }
+    });
+  });
+}
+
+function listLambdaVersions(
+  lambda: AWS.Lambda,
+  fnName: string
+): Promise<ListVersionsByFunctionResponse> {
+  return new Promise((resolve, reject) => {
+    lambda.listVersionsByFunction(
+      {
+        FunctionName: fnName,
+        MaxItems: 50
+      },
+      (err, res) => {
+        if (err) {
+          reject(err);
+        } else {
+          resolve(res);
+        }
+      }
+    );
+  });
+}
+
+function removeLambdaVersion(
+  lambda: AWS.Lambda,
+  fnName: string,
+  version: string
+): Promise<{}> {
+  return new Promise((resolve, reject) => {
+    lambda.deleteFunction(
+      { FunctionName: fnName, Qualifier: version },
+      (err, res) => {
+        if (err) {
+          reject(err);
+        } else {
+          resolve(res);
+        }
+      }
+    );
+  });
+}
+
+async function cleanupVersions(
+  region: string,
+  removeIt: boolean = false,
+  nextMarker: string | undefined = undefined
+) {
+  const lambda: AWS.Lambda = new AWS.Lambda({ region });
+  const lambdas = await listLambdaFunctions(lambda, nextMarker);
+
+  let i = 0;
+
+  for (const fn of lambdas.Functions ?? []) {
+    if (fn.FunctionName) {
+      const versions = await listLambdaVersions(lambda, fn.FunctionName);
+
+      for (const version of versions.Versions ?? []) {
+        if (version.Version && version.Version !== fn.Version) {
+          if (removeIt) {
+            try {
+              console.log(
+                `Removing function: ${fn.FunctionName} - ${version.Version}`
+              );
+              await removeLambdaVersion(
+                lambda,
+                fn.FunctionName,
+                version.Version
+              );
+              i++;
+            } catch (e) {
+              console.log(
+                `Remove failed (${fn.FunctionName} - ${version.Version}): ${e.message}`
+              );
+            }
+          } else {
+            console.log(
+              `Listing function only: ${fn.FunctionName} - ${version.Version}`
+            );
+            i++;
+          }
+        }
+      }
+    }
+  }
+
+  if (lambdas.NextMarker) {
+    i += await cleanupVersions(region, removeIt, lambdas.NextMarker);
+  }
+
+  return i;
+}
+
+console.info("Cleaning up old Lambda versions");
+cleanupVersions("us-east-1", true) // All Lambda@Edge is created in us-east-1 only
+  .then((success) => {
+    if (success) {
+      console.info("Cleaning up old Lambda versions successful.");
+      process.exit(0);
+    } else {
+      console.error("Cleaning up old Lambda versions failed.");
+      process.exit(1);
+    }
+  })
+  .catch((error) => {
+    console.error(`Unhandled error: ${error}`);
+    process.exit(1);
+  });

packages/e2e-tests/test-utils/scripts/run-e2e-ci.ts

@@ -256,6 +256,7 @@ function cleanup(): void {
     );
 
     // Optimistically clean up app's S3 bucket
+    console.info("Optimistically cleaning up app's S3 bucket");
     execSync(
       `aws s3 rm s3://${getAppBucketName(appName)} --recursive || true`,
       {

packages/e2e-tests/test-utils/tsconfig.scripts.json

@@ -0,0 +1,18 @@
+{
+  "compilerOptions": {
+    "baseUrl": "scripts",
+    "noImplicitReturns": true,
+    "noImplicitThis": true,
+    "noUnusedLocals": false,
+    "preserveConstEnums": true,
+    "removeComments": true,
+    "sourceMap": true,
+    "strictNullChecks": true,
+    "strict": true,
+    "esModuleInterop": true,
+    "target": "es2015",
+    "module": "commonjs",
+    "rootDir": "./"
+  },
+  "include": ["**/*.ts"]
+}