feat(fetch-http-handler): improve performance, replace FileReader with Blob.arrayBuffer() (#1179)

* Update stream-collector.ts

* Update stream collector

* Add changeset

* remove FileReader where possible

---------

Co-authored-by: Richard Davison <[email protected]>
Co-authored-by: George Fu <[email protected]>

Author: 3 people

.changeset/breezy-cobras-repair.md

@@ -0,0 +1,7 @@
+---
+"@smithy/chunked-blob-reader": major
+"@smithy/fetch-http-handler": major
+"@smithy/chunked-blob-reader-native": patch
+---
+
+replace FileReader with Blob.arrayBuffer() where possible

packages/chunked-blob-reader-native/src/index.ts

@@ -8,6 +8,10 @@ export function blobReader(
   chunkSize: number = 1024 * 1024
 ): Promise<void> {
   return new Promise((resolve, reject) => {
+    /**
+     * TODO(react-native): https://github.com/facebook/react-native/issues/34402
+     * To drop FileReader in react-native, we need the Blob.arrayBuffer() method to work.
+     */
     const fileReader = new FileReader();
 
     fileReader.onerror = reject;

packages/chunked-blob-reader/src/index.spec.ts

@@ -1,5 +1,10 @@
+import { Blob as BlobPolyfill } from "buffer";
+
 import { blobReader } from "./index";
 
+// jsdom inaccurate Blob https://github.com/jsdom/jsdom/issues/2555.
+global.Blob = BlobPolyfill as any;
+
 describe("blobReader", () => {
   it("reads an entire blob", async () => {
     const longMessage: number[] = [];

packages/chunked-blob-reader/src/index.ts

@@ -1,37 +1,18 @@
 /**
  * @internal
+ * Reads the blob data into the onChunk consumer.
  */
-export function blobReader(
+export async function blobReader(
   blob: Blob,
   onChunk: (chunk: Uint8Array) => void,
   chunkSize: number = 1024 * 1024
 ): Promise<void> {
-  return new Promise((resolve, reject) => {
-    const fileReader = new FileReader();
+  const size = blob.size;
+  let totalBytesRead = 0;
 
-    fileReader.addEventListener("error", reject);
-    fileReader.addEventListener("abort", reject);
-
-    const size = blob.size;
-    let totalBytesRead = 0;
-
-    function read() {
-      if (totalBytesRead >= size) {
-        resolve();
-        return;
-      }
-      fileReader.readAsArrayBuffer(blob.slice(totalBytesRead, Math.min(size, totalBytesRead + chunkSize)));
-    }
-
-    fileReader.addEventListener("load", (event) => {
-      const result = <ArrayBuffer>(event.target as any).result;
-      onChunk(new Uint8Array(result));
-      totalBytesRead += result.byteLength;
-      // read the next block
-      read();
-    });
-
-    // kick off the read
-    read();
-  });
+  while (totalBytesRead < size) {
+    const slice: Blob = blob.slice(totalBytesRead, Math.min(size, totalBytesRead + chunkSize));
+    onChunk(new Uint8Array(await slice.arrayBuffer()));
+    totalBytesRead += slice.size;
+  }
 }

packages/fetch-http-handler/src/stream-collector.spec.ts

@@ -1,36 +1,29 @@
+import { Blob as BlobPolyfill } from "buffer";
+
 import { streamCollector } from "./stream-collector";
 
-/**
- * Have to mock the FileReader behavior in IE, where
- * reader.result is null if reads an empty blob.
- */
+// jsdom inaccurate Blob https://github.com/jsdom/jsdom/issues/2555.
+global.Blob = BlobPolyfill as any;
+
 describe("streamCollector", () => {
-  let originalFileReader = (global as any).FileReader;
-  let originalBlob = (global as any).Blob;
-  beforeAll(() => {
-    originalFileReader = (global as any).FileReader;
-    originalBlob = (global as any).Blob;
-  });
-  afterAll(() => {
-    (global as any).FileReader = originalFileReader;
-    (global as any).Blob = originalBlob;
+  const blobAvailable = typeof Blob === "function";
+  const readableStreamAvailable = typeof ReadableStream === "function";
+
+  (blobAvailable ? it : it.skip)("collects Blob into bytearray", async () => {
+    const blobby = new Blob([new Uint8Array([1, 2]), new Uint8Array([3, 4])]);
+    const collected = await streamCollector(blobby);
+    expect(collected).toEqual(new Uint8Array([1, 2, 3, 4]));
   });
 
-  it("returns a Uint8Array when blob is empty and when FileReader data is null(in IE)", (done) => {
-    (global as any).FileReader = function FileReader() {
-      this.result = null; //In IE, FileReader.result is null after reading empty blob
-      this.readAsDataURL = jest.fn().mockImplementation(() => {
-        if (this.onloadend) {
-          this.readyState = 2;
-          this.onloadend();
-        }
-      });
-    };
-    (global as any).Blob = function Blob() {};
-    const dataPromise = streamCollector(new Blob());
-    dataPromise.then((data: any) => {
-      expect(data).toEqual(Uint8Array.from([]));
-      done();
+  (readableStreamAvailable ? it : it.skip)("collects ReadableStream into bytearray", async () => {
+    const stream = new ReadableStream({
+      start(controller) {
+        controller.enqueue(new Uint8Array([1, 2]));
+        controller.enqueue(new Uint8Array([3, 4]));
+        controller.close();
+      },
     });
+    const collected = await streamCollector(stream);
+    expect(collected).toEqual(new Uint8Array([1, 2, 3, 4]));
   });
 });

packages/fetch-http-handler/src/stream-collector.ts

@@ -1,21 +1,13 @@
 import { StreamCollector } from "@smithy/types";
-import { fromBase64 } from "@smithy/util-base64";
 
-//reference: https://snack.expo.io/r1JCSWRGU
-export const streamCollector: StreamCollector = (stream: Blob | ReadableStream): Promise<Uint8Array> => {
+export const streamCollector: StreamCollector = async (stream: Blob | ReadableStream): Promise<Uint8Array> => {
   if (typeof Blob === "function" && stream instanceof Blob) {
-    return collectBlob(stream);
+    return new Uint8Array(await stream.arrayBuffer());
   }
 
   return collectStream(stream as ReadableStream);
 };
 
-async function collectBlob(blob: Blob): Promise<Uint8Array> {
-  const base64 = await readToBase64(blob);
-  const arrayBuffer = fromBase64(base64);
-  return new Uint8Array(arrayBuffer);
-}
-
 async function collectStream(stream: ReadableStream): Promise<Uint8Array> {
   const chunks = [];
   const reader = stream.getReader();
@@ -40,26 +32,3 @@ async function collectStream(stream: ReadableStream): Promise<Uint8Array> {
 
   return collected;
 }
-
-function readToBase64(blob: Blob): Promise<string> {
-  return new Promise((resolve, reject) => {
-    const reader = new FileReader();
-    reader.onloadend = () => {
-      // reference: https://developer.mozilla.org/en-US/docs/Web/API/FileReader/readAsDataURL
-      // response from readAsDataURL is always prepended with "data:*/*;base64,"
-      if (reader.readyState !== 2) {
-        return reject(new Error("Reader aborted too early"));
-      }
-      const result = (reader.result ?? "") as string;
-      // Response can include only 'data:' for empty blob, return empty string in this case.
-      // Otherwise, return the string after ','
-      const commaIndex = result.indexOf(",");
-      const dataOffset = commaIndex > -1 ? commaIndex + 1 : result.length;
-      resolve(result.substring(dataOffset));
-    };
-    reader.onabort = () => reject(new Error("Read aborted"));
-    reader.onerror = () => reject(reader.error);
-    // reader.readAsArrayBuffer is not always available
-    reader.readAsDataURL(blob);
-  });
-}