feat: add support for browser streaming (#721)

Author: AllanZhengYP

jest.config.js

@@ -3,15 +3,9 @@ const base = require("./jest.config.base.js");
 module.exports = {
   ...base,
   projects: ["<rootDir>/packages/*/jest.config.js"],
-  testPathIgnorePatterns: [
-    "<rootDir>/packages/add-glacier-checksum-headers-browser",
-    "<rootDir>/clients/client-.*"
-  ],
+  testPathIgnorePatterns: ["/node_modules/", "<rootDir>/clients/client-.*"],
   coveragePathIgnorePatterns: [
     "/node_modules/",
-    "<rootDir>/packages/add-glacier-checksum-headers-browser",
-    "<rootDir>/packages/crypto-sjcl-*",
-    "<rootDir>/packages/xml-body-parser/vendor/",
     "<rootDir>/clients/client-.*",
     "/__fixtures__/"
   ]

package.json

@@ -14,7 +14,7 @@
     "build:smithy-client": "yarn build:crypto-dependencies && lerna run --scope '@aws-sdk/client-rds-data' --include-dependencies pretest",
     "build:all": "yarn build:crypto-dependencies && lerna run pretest --include-dependencies --include-dependents",
     "pretest:all": "yarn build:all",
-    "test:all": "jest --coverage --passWithNoTests",
+    "test:all": "jest --coverage --passWithNoTests && lerna run test --scope @aws-sdk/stream-collector-browser --scope @aws-sdk/hash-blob-browser",
     "test:functional": "jest --config tests/functional/jest.config.js --passWithNoTests",
     "test:integration": "cucumber.js"
   },
@@ -50,10 +50,16 @@
     "typescript": "^3.7.0",
     "yarn": "1.21.1"
   },
-  "workspaces": [
-    "packages/*",
-    "clients/*"
-  ],
+  "workspaces": {
+    "packages": [
+      "packages/*",
+      "clients/*"
+    ],
+    "nohoist": [
+      "**/karma*",
+      "**/karma*/**"
+    ]
+  },
   "husky": {
     "hooks": {
       "pre-commit": "lint-staged",

packages/fetch-http-handler/src/fetch-http-handler.spec.ts

@@ -40,7 +40,7 @@ describe("httpHandler", () => {
           ["bizz", "bazz"]
         ])
       },
-      blob: jest.fn().mockResolvedValue("")
+      body: "FOO" //should be a ReadableStream in real life.
     };
     const mockFetch = jest.fn().mockResolvedValue(mockResponse);
 
@@ -50,7 +50,7 @@ describe("httpHandler", () => {
     let response = await fetchHttpHandler.handle({} as any, {});
 
     expect(mockFetch.mock.calls.length).toBe(1);
-    expect(mockResponse.blob.mock.calls.length).toBe(1);
+    expect(response.response.body).toBe("FOO");
   });
 
   it("properly constructs url", async () => {
@@ -61,7 +61,7 @@ describe("httpHandler", () => {
           ["bizz", "bazz"]
         ])
       },
-      blob: jest.fn().mockResolvedValue("")
+      body: ""
     };
     const mockFetch = jest.fn().mockResolvedValue(mockResponse);
 

packages/fetch-http-handler/src/fetch-http-handler.ts

@@ -72,13 +72,13 @@ export class FetchHttpHandler implements HttpHandler {
           transformedHeaders[pair[0]] = pair[1];
         }
 
-        return response.blob().then<{ response: HttpResponse }>(body => ({
+        return {
           response: new HttpResponse({
             headers: transformedHeaders,
             statusCode: response.status,
-            body
+            body: response.body
           })
-        }));
+        };
       }),
       requestTimeout(requestTimeoutInMs)
     ];

packages/hash-blob-browser/karma.conf.js

@@ -48,9 +48,13 @@ module.exports = function(config) {
     // enable / disable watching file and executing tests whenever any file changes
     autoWatch: false,
 
-    // start these browsers
-    // available browser launchers: https://npmjs.org/browse/keyword/karma-launcher
-    browsers: ["ChromeHeadless"],
+    browsers: ["ChromeHeadlessNoSandbox"],
+    customLaunchers: {
+      ChromeHeadlessNoSandbox: {
+        base: "ChromeHeadless",
+        flags: ["--no-sandbox"]
+      }
+    },
 
     // Continuous Integration mode
     // if true, Karma captures browsers, runs the tests and exits

packages/stream-collector-browser/jest.config.js

@@ -0,0 +1,26 @@
+process.env.CHROME_BIN = require("puppeteer").executablePath();
+module.exports = function(config) {
+  config.set({
+    frameworks: ["jasmine", "karma-typescript"],
+    files: ["src/**/*.ts"],
+    exclude: ["**/*.d.ts"],
+    preprocessors: {
+      "**/*.ts": "karma-typescript"
+    },
+    reporters: ["progress", "karma-typescript"],
+    browsers: ["ChromeHeadlessNoSandbox"],
+    customLaunchers: {
+      ChromeHeadlessNoSandbox: {
+        base: "ChromeHeadless",
+        flags: ["--no-sandbox"]
+      }
+    },
+    karmaTypescriptConfig: {
+      tsconfig: "./tsconfig.json",
+      bundlerOptions: {
+        addNodeGlobals: false
+      }
+    },
+    singleRun: true
+  });
+};

packages/stream-collector-browser/karma.conf.js

@@ -4,8 +4,8 @@
   "description": "Provides a way to store the contents of a stream into a Uint8Array",
   "scripts": {
     "prepublishOnly": "tsc",
-    "pretest": "tsc -p tsconfig.test.json",
-    "test": "jest --coverage"
+    "pretest": "tsc -p tsconfig.json",
+    "test": "karma start karma.conf.js"
   },
   "author": {
     "name": "AWS SDK for JavaScript Team",
@@ -20,7 +20,13 @@
   },
   "devDependencies": {
     "@types/jest": "^24.0.12",
-    "jest": "^24.7.1",
+    "jasmine-core": "^3.5.0",
+    "karma": "^4.4.1",
+    "karma-chrome-launcher": "^3.1.0",
+    "karma-jasmine": "^2.0.1",
+    "karma-typescript": "^4.1.1",
+    "puppeteer": "^2.0.0",
+    "source-map": "^0.7.3",
     "typescript": "~3.4.0"
   }
 }

packages/stream-collector-browser/package.json

@@ -3,48 +3,22 @@ import { streamCollector } from "./index";
 declare const global: any;
 
 describe("streamCollector", () => {
-  const ambientFileReader =
-    typeof FileReader !== "undefined" ? FileReader : undefined;
-
-  beforeEach(() => {
-    const mockFileReader: FileReader = {
-      readAsArrayBuffer: jest.fn()
-    } as any;
-    global.FileReader = function() {
-      return mockFileReader;
-    } as any;
-  });
-
-  afterEach(() => {
-    global.FileReader = ambientFileReader as any;
-  });
-
-  it("returns a Uint8Array from a blob", async () => {
-    const dataPromise = streamCollector(new Blob());
-
-    const reader = new FileReader();
-    (reader as any).result = Uint8Array.from([0xde, 0xad]).buffer;
-    reader.onload!({} as any);
-
-    expect(await dataPromise).toEqual(Uint8Array.from([0xde, 0xad]));
+  it("returns a Uint8Array from a blob", done => {
+    const dataPromise = streamCollector(
+      new Response(new Uint8Array([102, 111, 111]).buffer).body
+    );
+    dataPromise.then((data: any) => {
+      expect(data).toEqual(Uint8Array.from([102, 111, 111]));
+      done();
+    });
   });
 
-  it("propagates errors encountered by the file reader", async () => {
-    const dataPromise = streamCollector(new Blob());
-
-    const reader = new FileReader();
-    (reader as any).error = new Error("PANIC");
-    reader.onerror!({} as any);
-
-    await expect(dataPromise).rejects.toMatchObject(new Error("PANIC"));
-  });
-
-  it("rejects the promise when the read is aborted", async () => {
-    const dataPromise = streamCollector(new Blob());
-
-    const reader = new FileReader();
-    reader.onabort!({} as any);
-
-    await expect(dataPromise).rejects.toMatchObject(new Error("Read aborted"));
+  it("returns a Uint8Array when stream is empty", done => {
+    const expected = new Uint8Array(0);
+    const dataPromise = streamCollector(new Response(expected.buffer).body);
+    dataPromise.then((data: any) => {
+      expect(data).toEqual(expected);
+      done();
+    });
   });
 });

packages/stream-collector-browser/src/index.spec.ts

@@ -1,12 +1,9 @@
 import { StreamCollector } from "@aws-sdk/types";
 
 export const streamCollector: StreamCollector = (
-  stream: Blob
-): Promise<Uint8Array> =>
-  new Promise<Uint8Array>((resolve, reject) => {
-    const reader = new FileReader();
-    reader.onload = () => resolve(new Uint8Array(reader.result as ArrayBuffer));
-    reader.onabort = () => reject(new Error("Read aborted"));
-    reader.onerror = () => reject(reader.error);
-    reader.readAsArrayBuffer(stream);
-  });
+  stream: ReadableStream
+): Promise<Uint8Array> => {
+  return new Response(stream)
+    .arrayBuffer()
+    .then(arrayBuffer => new Uint8Array(arrayBuffer));
+};

packages/stream-collector-browser/src/index.ts

@@ -37,7 +37,7 @@ export interface HashConstructor {
  * will consume the stream, so only replayable streams should be provided to an
  * implementation of this interface.
  */
-export interface StreamHasher<StreamType> {
+export interface StreamHasher<StreamType = any> {
   (hashCtor: { new (): Hash }, stream: StreamType): Promise<Uint8Array>;
 }