feat(lib-storage): rewrite lib-storage upload (#2039)

* fix: proper dependencies and node/browser configuration

* feat: adding example code for upload usage

* feat: export upload and upload types

* fix: example code imports the @aws-sdk/lib-storage package

* add tests for upload

* feat: rewrite functions that get chunks of buffers, readable, and readableStreams

* feat: add chunker wrapper that properly sends data to the correct chunker

* feat: add upload types

* feat: add main uploader that manages multipart upload steps

* feat: add best attempt at getting the byte size of an object

* fix: remove old files as part of rewrite

* Fix: add support for tags

* fix: only concat buffers when there is more than one buffer

* fix: grammatical and naming fixes to address feedback

* fix: correct the max number of parts s3 allows for multipart upload to 10000

* fix: use const in place of let

* fix: calling abort should throw abort error from upload

Co-authored-by: Trivikram Kamat <[email protected]>

* fix: fully enumerate upload options on readme

* fix: throw error on abort

* fix: addressing PR feedback

* fix: manually ran prettier

* fix: nit single line branch

* fix: remove unneeded add

* Fix: PR feedback

Co-authored-by: Trivikram Kamat <[email protected]>

Author: alexforsyth

lib/storage/README.md

@@ -1,18 +1,24 @@
 # @aws-sdk/lib-storage
 
-[![NPM version](https://img.shields.io/npm/v/@aws-sdk/lib-storage/latest.svg)](https://www.npmjs.com/package/@aws-sdk/abort-controller)
-[![NPM downloads](https://img.shields.io/npm/dm/@aws-sdk/lib-storage.svg)](https://www.npmjs.com/package/@aws-sdk/abort-controller)
+[![NPM version](https://img.shields.io/npm/v/@aws-sdk/lib-storage/latest.svg)](https://www.npmjs.com/package/@aws-sdk/lib-storage)
+[![NPM downloads](https://img.shields.io/npm/dm/@aws-sdk/lib-storage.svg)](https://www.npmjs.com/package/@aws-sdk/lib-storage)
 
 ### Upload
 
 Upload allows for easy and efficient uploading of buffers, blobs, or streams, using a configurable amount of concurrency to perform multipart uploads where possible. This abstraction enables uploading large files or streams of unknown size due to the use of multipart uploads under the hood.
 
 ```
   import { Upload } from "@aws-sdk/lib-storage";
+  import { S3Client, S3 } from "@aws-sdk/client-s3";
+
   const target = { Bucket, Key, Body };
   try {
     const paralellUploads3 = new Upload({
-      client: new S3({}),
+      client: new S3({}) || new S3Client({}),
+      tags: [...], // optional tags
+      queueSize: 4, // optional concurrency configuration
+      partSize: 5MB, // optional size of each part
+      leavePartsOnError: false, // optional manually handle dropped parts
       params: target,
     });
 

lib/storage/example-code/config.ts

@@ -0,0 +1,4 @@
+export const configuration = {
+  Bucket: "test-bucket-1234567890",
+  Key: `unique-key-${Date.now}`,
+};

lib/storage/example-code/file-upload.ts

@@ -1,29 +1,24 @@
-import { Upload } from "../src/index";
+import { Upload } from "@aws-sdk/lib-storage";
 import { S3 } from "@aws-sdk/client-s3";
+import { configuration } from "./config";
 
 const fs = require("fs");
-const fileStream = fs.createReadStream("./big.file");
+const fileStream = fs.createReadStream(__dirname + "/big.file");
 
 (async () => {
-  const target = {
-    Bucket: "aws-sdk-js-mock-files",
-    Key: "data_key",
-    Body: fileStream,
-  };
+  const upload = new Upload({
+    params: {
+      Bucket: configuration.Bucket,
+      Key: configuration.Key,
+      Body: fileStream,
+    },
+    client: new S3({}),
+    queueSize: 3,
+  });
 
-  try {
-    const upload = new Upload({
-      params: target,
-      client: new S3({}),
-      queueSize: 3,
-    });
+  upload.on("httpUploadProgress", (progress) => {
+    console.log(progress);
+  });
 
-    upload.on("httpUploadProgress", (progress) => {
-      console.log(progress);
-    });
-
-    await upload.done();
-  } catch (e) {
-    console.log(e);
-  }
+  await upload.done();
 })();

lib/storage/example-code/upload-abort.ts

@@ -0,0 +1,48 @@
+import { S3Client } from "@aws-sdk/client-s3";
+import { Upload } from "@aws-sdk/lib-storage";
+import { configuration } from "./config";
+import { Readable } from "stream";
+
+const Bucket = configuration.Bucket;
+const region = "us-west-2";
+
+const sleep = async (seconds: number) => {
+  return new Promise((resolve) => setTimeout(resolve, seconds * 1000));
+};
+
+async function* generateContents() {
+  for (let index = 0; index < 8; index++) {
+    const time = Math.random() * 5;
+    await sleep(time);
+    console.log(`Delaying part ${index} for ${time}`);
+    yield `[Part ${index}] ${"#".repeat(2000000)}`;
+  }
+}
+
+const uploadIndeterminateLengthStreamNode = async () => {
+  const streamOfUnknownlength = Readable.from(generateContents());
+
+  const Key = configuration.Key;
+  let upload = new Upload({
+    client: new S3Client({ region }),
+    params: {
+      Key,
+      Bucket,
+      Body: streamOfUnknownlength,
+    },
+  });
+
+  upload.on("httpUploadProgress", (progress: ProgressEvent) => {
+    console.log(progress);
+  });
+
+  setTimeout(() => {
+    console.log(" Aborting ....");
+    let res = upload.abort();
+  }, 10 * 1000);
+
+  const uploadResult = await upload.done();
+  console.log("done uploading", uploadResult);
+};
+
+uploadIndeterminateLengthStreamNode();

lib/storage/example-code/upload-string.ts

@@ -1,40 +1,22 @@
-import { Upload } from "../src/index";
-import { S3, S3Client } from "@aws-sdk/client-s3";
+import { Upload } from "@aws-sdk/lib-storage";
+import { S3 } from "@aws-sdk/client-s3";
+import { configuration } from "./config";
 
-const Bucket = "aws-sdk-js-mock-files";
-const Key = "data_key";
+const Bucket = configuration.Bucket;
+const Key = configuration.Key;
 const Body =
   "Duo Reges: constructio interrete. Qui autem esse poteris, nisi te amor ipse ceperit? Hoc est non modo cor non habere, sed ne palatum quidem. Quantam rem agas, ut Circeis qui habitet totum hunc mundum suum municipium esse existimet? Huius, Lyco, oratione locuples, rebus ipsis ielunior. Sed quid attinet de rebus tam apertis plura requirere? Non quam nostram quidem, inquit Pomponius iocans; Alterum significari idem, ut si diceretur, officia media omnia aut pleraque servantem vivere. Stoici scilicet";
 
 (async () => {
   const target = { Bucket, Key, Body };
-  try {
-    const paralellUploads3 = new Upload({
-      client: new S3({}),
-      params: target,
-    });
+  const paralellUploads3 = new Upload({
+    client: new S3({}),
+    params: target,
+  });
 
-    paralellUploads3.on("httpUploadProgress", (progress) => {
-      console.log(progress);
-    });
+  paralellUploads3.on("httpUploadProgress", (progress) => {
+    console.log(progress);
+  });
 
-    await paralellUploads3.done();
-  } catch (e) {
-    console.log(e);
-  }
-
-  try {
-    const paralellUploads3Client = new Upload({
-      client: new S3Client({}),
-      params: target,
-    });
-
-    paralellUploads3Client.on("httpUploadProgress", (progress) => {
-      console.log(progress);
-    });
-
-    await paralellUploads3Client.done();
-  } catch (e) {
-    console.log(e);
-  }
+  await paralellUploads3.done();
 })();

lib/storage/example-code/upload-unknown-length-browser.ts

@@ -0,0 +1,57 @@
+import { fromCognitoIdentityPool } from "@aws-sdk/credential-provider-cognito-identity";
+import { CognitoIdentityClient } from "@aws-sdk/client-cognito-identity";
+
+import { S3Client } from "@aws-sdk/client-s3";
+import { Upload } from "@aws-sdk/lib-storage";
+
+import { configuration } from "./config";
+
+const idPool = "us-west-2:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX";
+const Bucket = configuration.Bucket;
+const region = "us-west-2";
+
+const sleep = async (seconds: number) => {
+  return new Promise((resolve) => setTimeout(resolve, seconds * 1000));
+};
+
+const fakeStreamOfUnknownlength = new ReadableStream({
+  start: async (controller) => {
+    for (let index = 0; index < 8; index++) {
+      const time = Math.random() * 5;
+      await sleep(time);
+      console.log(`Delaying part ${index} for ${time}`);
+      controller.enqueue(`[part ${index}] with ${"#".repeat(2000000)}`);
+    }
+    controller.close();
+  },
+});
+
+const uploadIndeterminateLengthStreamBrowser = async () => {
+  const client = new S3Client({
+    region,
+    credentials: fromCognitoIdentityPool({
+      client: new CognitoIdentityClient({ region }),
+      identityPoolId: idPool,
+    }),
+  });
+
+  const Key = configuration.Key;
+
+  let upload = new Upload({
+    client,
+    params: {
+      Key,
+      Bucket,
+      Body: fakeStreamOfUnknownlength,
+    },
+  });
+
+  upload.on("httpUploadProgress", (progress: ProgressEvent) => {
+    console.log(progress);
+  });
+
+  const uploadResult = await upload.done();
+  console.log("done uploading", uploadResult);
+};
+
+uploadIndeterminateLengthStreamBrowser();

lib/storage/example-code/upload-unknown-length.ts

@@ -0,0 +1,43 @@
+import { S3Client } from "@aws-sdk/client-s3";
+import { Upload } from "@aws-sdk/lib-storage";
+
+import { Readable } from "stream";
+import { configuration } from "./config";
+
+const Bucket = configuration.Bucket;
+const region = "us-west-2";
+
+const sleep = async (seconds: number) => {
+  return new Promise((resolve) => setTimeout(resolve, seconds * 1000));
+};
+
+async function* generateContents() {
+  for (let index = 0; index < 8; index++) {
+    const time = Math.random() * 10;
+    await sleep(time);
+    console.log(`Delaying part ${index} for ${time}`);
+    yield `[Part ${index}] ${"#".repeat(2000000)}`;
+  }
+}
+const fakeStreamOfUnknownlength = Readable.from(generateContents());
+
+const uploadIndeterminateLengthStreamNode = async () => {
+  const Key = configuration.Key;
+  let upload = new Upload({
+    client: new S3Client({ region }),
+    params: {
+      Key,
+      Bucket,
+      Body: fakeStreamOfUnknownlength,
+    },
+  });
+
+  upload.on("httpUploadProgress", (progress: ProgressEvent) => {
+    console.log(progress);
+  });
+
+  const uploadResult = await upload.done();
+  console.log("done uploading", uploadResult);
+};
+
+uploadIndeterminateLengthStreamNode();

lib/storage/package.json

@@ -23,11 +23,11 @@
   },
   "license": "Apache-2.0",
   "dependencies": {
+    "@aws-sdk/abort-controller": "3.4.1",
     "@aws-sdk/client-s3": "3.5.0",
     "buffer": "^5.6.0",
     "stream-browserify": "^3.0.0",
-    "tslib": "^1.8.0",
-    "web-streams-polyfill": "^3.0.0"
+    "tslib": "^1.8.0"
   },
   "devDependencies": {
     "@types/jest": "^26.0.4",
@@ -40,7 +40,8 @@
     "karma-spec-reporter": "^0.0.32",
     "karma-typescript": "^5.2.0",
     "ts-jest": "^26.4.1",
-    "typescript": "~4.1.2"
+    "typescript": "~4.1.2",
+    "web-streams-polyfill": "^3.0.0"
   },
   "typesVersions": {
     "<4.0": {
@@ -49,6 +50,14 @@
       ]
     }
   },
+  "browser": {
+    "stream": "stream-browserify",
+    "fs": "./src/runtimeConfig.browser",
+    "./runtimeConfig": "./src/runtimeConfig.browser"
+  },
+  "react-native": {
+    "./runtimeConfig": "./src/runtimeConfig.native"
+  },
   "homepage": "https://github.com/trivikr/aws-sdk-js-v3/tree/master/lib/storage",
   "repository": {
     "type": "git",