fix(syncer): replaced aws-sdk v2 by aws-sdk v3 (#3075)

* fix: update aws sdk v3

* increase coverage thresholds

* remove tslog

Author: npalm

Diff for: modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/jest.config.js

@@ -5,10 +5,10 @@ module.exports = {
   collectCoverageFrom: ['src/**/*.{ts,js,jsx}', '!src/**/*local*.ts', '!src/**/*.d.ts'],
   coverageThreshold: {
     global: {
-      branches: 85,
-      functions: 78,
-      lines: 93,
-      statements: 93
+      branches: 87,
+      functions: 92,
+      lines: 98,
+      statements: 98
     }
   }
 };

Diff for: modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/package.json

@@ -26,6 +26,8 @@
     "@typescript-eslint/eslint-plugin": "^5.54.1",
     "@typescript-eslint/parser": "^5.54.1",
     "@vercel/ncc": "^0.36.1",
+    "aws-sdk-client-mock": "^2.1.1",
+    "aws-sdk-client-mock-jest": "^2.1.1",
     "eslint": "^8.35.0",
     "eslint-plugin-prettier": "4.2.1",
     "jest": "^29.5",
@@ -37,8 +39,8 @@
   },
   "dependencies": {
     "@aws-lambda-powertools/logger": "^1.6.0",
-    "aws-sdk": "^2.1312.0",
-    "axios": "^1.3.4",
-    "tslog": "^3.3.4"
+    "@aws-sdk/client-s3": "^3.294.0",
+    "@aws-sdk/lib-storage": "^3.294.0",
+    "axios": "^1.3.4"
   }
 }

Diff for: modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/src/syncer/syncer.test.ts

@@ -1,3 +1,6 @@
+import { GetObjectTaggingCommand, PutObjectCommand, S3Client } from '@aws-sdk/client-s3';
+import { mockClient } from 'aws-sdk-client-mock';
+import 'aws-sdk-client-mock-jest';
 import axios from 'axios';
 import { PassThrough } from 'stream';
 
@@ -21,20 +24,13 @@ mockStream.push(mockResponse);
 mockStream.end();
 
 jest.mock('axios');
-const mockedAxios = axios as jest.Mocked<typeof axios>;
-mockedAxios.request.mockResolvedValue({
+const mockAxios = axios as jest.Mocked<typeof axios>;
+mockAxios.get.mockResolvedValue({
   data: mockStream,
 });
 
-const mockS3 = {
-  getObjectTagging: jest.fn(),
-  upload: jest.fn().mockImplementation(() => {
-    return { promise: jest.fn(() => Promise.resolve()) };
-  }),
-};
-jest.mock('aws-sdk', () => ({
-  S3: jest.fn().mockImplementation(() => mockS3),
-}));
+process.env.AWS_REGION = 'us-east-1';
+const mockS3client = mockClient(S3Client);
 
 const bucketName = 'my-bucket';
 const objectExtension: Record<string, string> = {
@@ -54,14 +50,14 @@ const latestRelease = '2.296.2';
 
 beforeEach(() => {
   jest.clearAllMocks();
+  mockS3client.reset();
 });
 
 jest.setTimeout(60 * 1000);
 
 describe('Synchronize action distribution (no S3 tags).', () => {
   beforeEach(() => {
     process.env.S3_BUCKET_NAME = bucketName;
-
     mockOctokit.repos.getLatestRelease.mockImplementation(() => ({
       data: mockDataLatestRelease,
     }));
@@ -70,25 +66,19 @@ describe('Synchronize action distribution (no S3 tags).', () => {
   test.each(runnerOs)('%p Distribution is S3 has no tags.', async (os) => {
     process.env.S3_OBJECT_KEY = bucketObjectKey(os);
     process.env.GITHUB_RUNNER_OS = os;
-    mockS3.getObjectTagging.mockImplementation(() => {
-      return {
-        promise() {
-          return Promise.resolve({
-            TagSet: undefined,
-          });
-        },
-      };
+    mockS3client.on(GetObjectTaggingCommand).resolves({
+      TagSet: undefined,
     });
 
     await sync();
-    expect(mockS3.upload).toBeCalledTimes(1);
+    expect(mockS3client).toHaveReceivedCommandTimes(PutObjectCommand, 1);
   });
 });
 
-describe('Synchronize action distribution (up-to-date).', () => {
+describe('Synchronize action distribution.', () => {
   beforeEach(() => {
     process.env.S3_BUCKET_NAME = bucketName;
-
+    mockS3client.reset();
     mockOctokit.repos.getLatestRelease.mockImplementation(() => ({
       data: mockDataLatestRelease,
     }));
@@ -97,67 +87,81 @@ describe('Synchronize action distribution (up-to-date).', () => {
   test.each(runnerOs)('%p Distribution is up-to-date with latest release.', async (os) => {
     process.env.S3_OBJECT_KEY = bucketObjectKey(os);
     process.env.GITHUB_RUNNER_OS = os;
-    mockS3.getObjectTagging.mockImplementation(() => {
-      return {
-        promise() {
-          return Promise.resolve({
-            TagSet: [{ Key: 'name', Value: `actions-runner-${os}-x64-${latestRelease}${objectExtension[os]}` }],
-          });
-        },
-      };
+    mockS3client.on(GetObjectTaggingCommand).resolves({
+      TagSet: [{ Key: 'name', Value: `actions-runner-${os}-x64-${latestRelease}${objectExtension[os]}` }],
     });
 
     await sync();
     expect(mockOctokit.repos.getLatestRelease).toBeCalledTimes(1);
-    expect(mockS3.getObjectTagging).toBeCalledWith({
+    expect(mockS3client).toHaveReceivedNthCommandWith(1, GetObjectTaggingCommand, {
       Bucket: bucketName,
       Key: bucketObjectKey(os),
     });
-    expect(mockS3.upload).toBeCalledTimes(0);
+
+    expect(mockS3client).toHaveReceivedCommandTimes(PutObjectCommand, 0);
   });
 
   test.each(runnerOs)('%p Distribution should update to release.', async (os) => {
     process.env.S3_OBJECT_KEY = bucketObjectKey(os);
     process.env.GITHUB_RUNNER_OS = os;
-    mockS3.getObjectTagging.mockImplementation(() => {
-      return {
-        promise() {
-          return Promise.resolve({
-            TagSet: [{ Key: 'name', Value: `actions-runner-${os}-x64-0${objectExtension[os]}` }],
-          });
-        },
-      };
+
+    mockS3client.on(GetObjectTaggingCommand).resolves({
+      TagSet: [{ Key: 'name', Value: `actions-runner-${os}-x64-0${objectExtension[os]}` }],
     });
 
     await sync();
     expect(mockOctokit.repos.getLatestRelease).toBeCalledTimes(1);
-    expect(mockS3.getObjectTagging).toBeCalledWith({
+    expect(mockS3client).toHaveReceivedNthCommandWith(1, GetObjectTaggingCommand, {
       Bucket: bucketName,
       Key: bucketObjectKey(os),
     });
-    expect(mockS3.upload).toBeCalledTimes(1);
-    const s3JsonBody = mockS3.upload.mock.calls[0][0];
-    expect(s3JsonBody['Tagging']).toEqual(`name=actions-runner-${os}-x64-${latestRelease}${objectExtension[os]}`);
+
+    expect(mockS3client).toHaveReceivedNthSpecificCommandWith(1, PutObjectCommand, {
+      Bucket: bucketName,
+      Key: bucketObjectKey(os),
+      Tagging: `name=actions-runner-${os}-x64-${latestRelease}${objectExtension[os]}`,
+    });
+  });
+
+  test.each(runnerOs)('%p Distribution should update to release (tags look-up errored)', async (os) => {
+    process.env.S3_OBJECT_KEY = bucketObjectKey(os);
+    process.env.GITHUB_RUNNER_OS = os;
+
+    mockS3client.on(GetObjectTaggingCommand).rejects(new Error('No tags'));
+
+    await sync();
+    expect(mockOctokit.repos.getLatestRelease).toBeCalledTimes(1);
+    expect(mockS3client).toHaveReceivedNthCommandWith(1, GetObjectTaggingCommand, {
+      Bucket: bucketName,
+      Key: bucketObjectKey(os),
+    });
+
+    expect(mockS3client).toHaveReceivedNthSpecificCommandWith(1, PutObjectCommand, {
+      Bucket: bucketName,
+      Key: bucketObjectKey(os),
+      Tagging: `name=actions-runner-${os}-x64-${latestRelease}${objectExtension[os]}`,
+    });
   });
 
   test.each(runnerOs)('%p Tags, but no version, distribution should update.', async (os) => {
     process.env.S3_OBJECT_KEY = bucketObjectKey(os);
     process.env.GITHUB_RUNNER_OS = os;
-    mockS3.getObjectTagging.mockImplementation(() => {
-      return {
-        promise() {
-          return Promise.resolve({ TagSet: [{ Key: 'someKey', Value: 'someValue' }] });
-        },
-      };
+    mockS3client.on(GetObjectTaggingCommand).resolves({
+      TagSet: [{ Key: 'someKey', Value: `someValue` }],
     });
 
     await sync();
     expect(mockOctokit.repos.getLatestRelease).toBeCalledTimes(1);
-    expect(mockS3.getObjectTagging).toBeCalledWith({
+    expect(mockS3client).toHaveReceivedNthCommandWith(1, GetObjectTaggingCommand, {
+      Bucket: bucketName,
+      Key: bucketObjectKey(os),
+    });
+
+    expect(mockS3client).toHaveReceivedNthSpecificCommandWith(1, PutObjectCommand, {
       Bucket: bucketName,
       Key: bucketObjectKey(os),
+      Tagging: `name=actions-runner-${os}-x64-${latestRelease}${objectExtension[os]}`,
     });
-    expect(mockS3.upload).toBeCalledTimes(1);
   });
 });
 

Diff for: modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/src/syncer/syncer.ts

@@ -1,8 +1,8 @@
+import { GetObjectTaggingCommand, S3Client, Tag } from '@aws-sdk/client-s3';
+import { Upload } from '@aws-sdk/lib-storage';
 import { Octokit } from '@octokit/rest';
-import { S3 } from 'aws-sdk';
-import AWS from 'aws-sdk';
 import axios from 'axios';
-import { PassThrough } from 'stream';
+import { Stream } from 'stream';
 
 import { createChildLogger } from '../logger';
 
@@ -15,16 +15,16 @@ interface CacheObject {
   key: string;
 }
 
-async function getCachedVersion(s3: S3, cacheObject: CacheObject): Promise<string | undefined> {
+async function getCachedVersion(s3Client: S3Client, cacheObject: CacheObject): Promise<string | undefined> {
+  const command = new GetObjectTaggingCommand({
+    Bucket: cacheObject.bucket,
+    Key: cacheObject.key,
+  });
+
   try {
-    const objectTagging = await s3
-      .getObjectTagging({
-        Bucket: cacheObject.bucket,
-        Key: cacheObject.key,
-      })
-      .promise();
-    const versions = objectTagging.TagSet?.filter((t: S3.Tag) => t.Key === versionKey);
-    return versions.length === 1 ? versions[0].Value : undefined;
+    const objectTagging = await s3Client.send(command);
+    const versions = objectTagging.TagSet?.filter((t: Tag) => t.Key === versionKey);
+    return versions?.length === 1 ? versions[0].Value : undefined;
   } catch (e) {
     logger.debug('No tags found');
     return undefined;
@@ -53,48 +53,39 @@ async function getReleaseAsset(runnerOs = 'linux', runnerArch = 'x64'): Promise<
   return assets?.length === 1 ? { name: assets[0].name, downloadUrl: assets[0].browser_download_url } : undefined;
 }
 
-async function uploadToS3(s3: S3, cacheObject: CacheObject, actionRunnerReleaseAsset: ReleaseAsset): Promise<void> {
-  const writeStream = new PassThrough();
-  const writePromise = s3
-    .upload({
+async function uploadToS3(
+  s3Client: S3Client,
+  cacheObject: CacheObject,
+  actionRunnerReleaseAsset: ReleaseAsset,
+): Promise<void> {
+  const response = await axios.get(actionRunnerReleaseAsset.downloadUrl, {
+    responseType: 'stream',
+  });
+
+  const passThrough = new Stream.PassThrough();
+  response.data.pipe(passThrough);
+
+  const upload = new Upload({
+    client: s3Client,
+    params: {
       Bucket: cacheObject.bucket,
       Key: cacheObject.key,
       Tagging: versionKey + '=' + actionRunnerReleaseAsset.name,
-      Body: writeStream,
+      Body: passThrough,
       ServerSideEncryption: process.env.S3_SSE_ALGORITHM,
-      SSEKMSKeyId: process.env.S3_SSE_KMS_KEY_ID,
-    })
-    .promise();
-
-  logger.debug(`Start downloading ${actionRunnerReleaseAsset.name} and uploading to S3.`);
-
-  const readPromise = new Promise<void>((resolve, reject) => {
-    axios
-      .request<NodeJS.ReadableStream>({
-        method: 'get',
-        url: actionRunnerReleaseAsset.downloadUrl,
-        responseType: 'stream',
-      })
-      .then((res) => {
-        res.data
-          .pipe(writeStream)
-
-          .on('finish', () => resolve())
-          .on('error', (error) => reject(error));
-      })
-      .catch((error) => reject(error));
+    },
   });
 
-  await Promise.all([readPromise, writePromise])
-    .then(() => logger.info(`The new distribution is uploaded to S3.`))
-    .catch((error) => {
-      logger.error('Uploading of the new distribution to S3 failed.', error);
-      throw error;
-    });
+  upload.on('httpUploadProgress', () => logger.debug(`Downloading ${actionRunnerReleaseAsset.name} in progress`));
+  logger.debug(`Start downloading ${actionRunnerReleaseAsset.name} and uploading to S3.`);
+  await upload
+    .done()
+    .then(() => logger.info(`The new distribution ${actionRunnerReleaseAsset.name} is uploaded to S3.`))
+    .catch((e) => logger.error(`Error uploading ${actionRunnerReleaseAsset.name} to S3`, e));
 }
 
 export async function sync(): Promise<void> {
-  const s3 = new AWS.S3();
+  const s3 = new S3Client({});
 
   const runnerOs = process.env.GITHUB_RUNNER_OS || 'linux';
   const runnerArch = process.env.GITHUB_RUNNER_ARCHITECTURE || 'x64';