feat(client-s3): support generating endpoints from multi-region access point (#2796)

* Revert "fix(client-s3): revert MRAP customizations (#2759)"

This reverts commit 9f04714.

* chore: update mrap packages dependencies

* feat(middleware-sdk-s3): dynamically import aws-crt package optional dependencies

* feat(middleware-user-agent): track if aws-crt is available at runtime

* chore: update crt signer loading error message

* chore(middleware-sdk-s3): add crt package to dev dependency and track usage

This makes sure the type of CRT package is available and the Lerna builds
packages in right order--build crt signer package before buiding s3 middleware
package.

* docs(client-s3): add docs to Bucket members on using MRAP

* chore(signature-v4-crt): update crt version to ^1.9.7

* fix(client-s3): address PR feedbacks

Author: AllanZhengYP

packages/signature-v4/src/SignatureV4.ts

@@ -37,8 +37,9 @@ import { createScope, getSigningKey } from "./credentialDerivation";
 import { getCanonicalHeaders } from "./getCanonicalHeaders";
 import { getCanonicalQuery } from "./getCanonicalQuery";
 import { getPayloadHash } from "./getPayloadHash";
-import { hasHeader } from "./hasHeader";
+import { hasHeader } from "./headerUtil";
 import { moveHeadersToQuery } from "./moveHeadersToQuery";
+import { normalizeCredentialsProvider, normalizeRegionProvider } from "./normalizeProvider";
 import { prepareRequest } from "./prepareRequest";
 import { iso8601 } from "./utilDate";
 
@@ -317,21 +318,3 @@ const formatDate = (now: DateInput): { longDate: string; shortDate: string } =>
 };
 
 const getCanonicalHeaderList = (headers: object): string => Object.keys(headers).sort().join(";");
-
-const normalizeRegionProvider = (region: string | Provider<string>): Provider<string> => {
-  if (typeof region === "string") {
-    const promisified = Promise.resolve(region);
-    return () => promisified;
-  } else {
-    return region;
-  }
-};
-
-const normalizeCredentialsProvider = (credentials: Credentials | Provider<Credentials>): Provider<Credentials> => {
-  if (typeof credentials === "object") {
-    const promisified = Promise.resolve(credentials);
-    return () => promisified;
-  } else {
-    return credentials;
-  }
-};

packages/signature-v4/src/cloneRequest.ts

@@ -3,20 +3,17 @@ import { HttpRequest, QueryParameterBag } from "@aws-sdk/types";
 /**
  * @internal
  */
-export function cloneRequest({ headers, query, ...rest }: HttpRequest): HttpRequest {
-  return {
-    ...rest,
-    headers: { ...headers },
-    query: query ? cloneQuery(query) : undefined,
-  };
-}
+export const cloneRequest = ({ headers, query, ...rest }: HttpRequest): HttpRequest => ({
+  ...rest,
+  headers: { ...headers },
+  query: query ? cloneQuery(query) : undefined,
+});
 
-function cloneQuery(query: QueryParameterBag): QueryParameterBag {
-  return Object.keys(query).reduce((carry: QueryParameterBag, paramName: string) => {
+export const cloneQuery = (query: QueryParameterBag): QueryParameterBag =>
+  Object.keys(query).reduce((carry: QueryParameterBag, paramName: string) => {
     const param = query[paramName];
     return {
       ...carry,
       [paramName]: Array.isArray(param) ? [...param] : param,
     };
   }, {});
-}

packages/signature-v4/src/constants.ts

@@ -5,6 +5,7 @@ export const SIGNED_HEADERS_QUERY_PARAM = "X-Amz-SignedHeaders";
 export const EXPIRES_QUERY_PARAM = "X-Amz-Expires";
 export const SIGNATURE_QUERY_PARAM = "X-Amz-Signature";
 export const TOKEN_QUERY_PARAM = "X-Amz-Security-Token";
+export const REGION_SET_PARAM = "X-Amz-Region-Set";
 
 export const AUTH_HEADER = "authorization";
 export const AMZ_DATE_HEADER = AMZ_DATE_QUERY_PARAM.toLowerCase();
@@ -40,6 +41,7 @@ export const SEC_HEADER_PATTERN = /^sec-/;
 export const UNSIGNABLE_PATTERNS = [/^proxy-/i, /^sec-/i];
 
 export const ALGORITHM_IDENTIFIER = "AWS4-HMAC-SHA256";
+export const ALGORITHM_IDENTIFIER_V4A = "AWS4-ECDSA-P256-SHA256";
 
 export const EVENT_ALGORITHM_IDENTIFIER = "AWS4-HMAC-SHA256-PAYLOAD";
 

packages/signature-v4/src/credentialDerivation.ts

@@ -13,9 +13,8 @@ const cacheQueue: Array<string> = [];
  * @param region    The AWS region in which the service resides.
  * @param service   The service to which the signed request is being sent.
  */
-export function createScope(shortDate: string, region: string, service: string): string {
-  return `${shortDate}/${region}/${service}/${KEY_TYPE_IDENTIFIER}`;
-}
+export const createScope = (shortDate: string, region: string, service: string): string =>
+  `${shortDate}/${region}/${service}/${KEY_TYPE_IDENTIFIER}`;
 
 /**
  * Derive a signing key from its composite parts
@@ -57,15 +56,15 @@ export const getSigningKey = async (
 /**
  * @internal
  */
-export function clearCredentialCache(): void {
+export const clearCredentialCache = (): void => {
   cacheQueue.length = 0;
   Object.keys(signingKeyCache).forEach((cacheKey) => {
     delete signingKeyCache[cacheKey];
   });
-}
+};
 
-function hmac(ctor: HashConstructor, secret: SourceData, data: SourceData): Promise<Uint8Array> {
+const hmac = (ctor: HashConstructor, secret: SourceData, data: SourceData): Promise<Uint8Array> => {
   const hash = new ctor(secret);
   hash.update(data);
   return hash.digest();
-}
+};

packages/signature-v4/src/getCanonicalHeaders.ts

@@ -3,13 +3,13 @@ import { HeaderBag, HttpRequest } from "@aws-sdk/types";
 import { ALWAYS_UNSIGNABLE_HEADERS, PROXY_HEADER_PATTERN, SEC_HEADER_PATTERN } from "./constants";
 
 /**
- * @internal
+ * @private
  */
-export function getCanonicalHeaders(
+export const getCanonicalHeaders = (
   { headers }: HttpRequest,
   unsignableHeaders?: Set<string>,
   signableHeaders?: Set<string>
-): HeaderBag {
+): HeaderBag => {
   const canonical: HeaderBag = {};
   for (const headerName of Object.keys(headers).sort()) {
     const canonicalHeaderName = headerName.toLowerCase();
@@ -28,4 +28,4 @@ export function getCanonicalHeaders(
   }
 
   return canonical;
-}
+};

packages/signature-v4/src/getCanonicalQuery.ts

@@ -4,9 +4,9 @@ import { escapeUri } from "@aws-sdk/util-uri-escape";
 import { SIGNATURE_HEADER } from "./constants";
 
 /**
- * @internal
+ * @private
  */
-export function getCanonicalQuery({ query = {} }: HttpRequest): string {
+export const getCanonicalQuery = ({ query = {} }: HttpRequest): string => {
   const keys: Array<string> = [];
   const serialized: { [key: string]: string } = {};
   for (const key of Object.keys(query).sort()) {
@@ -34,4 +34,4 @@ export function getCanonicalQuery({ query = {} }: HttpRequest): string {
     .map((key) => serialized[key])
     .filter((serialized) => serialized) // omit any falsy values
     .join("&");
-}
+};

packages/signature-v4/src/getPayloadHash.ts

@@ -5,12 +5,12 @@ import { toHex } from "@aws-sdk/util-hex-encoding";
 import { SHA256_HEADER, UNSIGNED_PAYLOAD } from "./constants";
 
 /**
- * @internal
+ * @private
  */
-export async function getPayloadHash(
+export const getPayloadHash = async (
   { headers, body }: HttpRequest,
   hashConstructor: HashConstructor
-): Promise<string> {
+): Promise<string> => {
   for (const headerName of Object.keys(headers)) {
     if (headerName.toLowerCase() === SHA256_HEADER) {
       return headers[headerName];
@@ -29,4 +29,4 @@ export async function getPayloadHash(
   // body is unsignable. Attempt to send the request with an unsigned payload,
   // which may or may not be accepted by the service.
   return UNSIGNED_PAYLOAD;
-}
+};

packages/signature-v4/src/hasHeader.ts

@@ -0,0 +1,34 @@
+import { HeaderBag } from "@aws-sdk/types";
+
+export const hasHeader = (soughtHeader: string, headers: HeaderBag): boolean => {
+  soughtHeader = soughtHeader.toLowerCase();
+  for (const headerName of Object.keys(headers)) {
+    if (soughtHeader === headerName.toLowerCase()) {
+      return true;
+    }
+  }
+
+  return false;
+};
+
+/* Get the value of one request header, ignore the case. Return string if header is in the headers, else return undefined */
+export const getHeaderValue = (soughtHeader: string, headers: HeaderBag): string | undefined => {
+  soughtHeader = soughtHeader.toLowerCase();
+  for (const headerName of Object.keys(headers)) {
+    if (soughtHeader === headerName.toLowerCase()) {
+      return headers[headerName];
+    }
+  }
+
+  return undefined;
+};
+
+/* Delete the one request header, ignore the case. Do nothing if it's not there */
+export const deleteHeader = (soughtHeader: string, headers: HeaderBag) => {
+  soughtHeader = soughtHeader.toLowerCase();
+  for (const headerName of Object.keys(headers)) {
+    if (soughtHeader === headerName.toLowerCase()) {
+      delete headers[headerName];
+    }
+  }
+};

packages/signature-v4/src/headerUtil.ts

@@ -1,2 +1,8 @@
 export * from "./credentialDerivation";
+export { getCanonicalHeaders } from "./getCanonicalHeaders";
+export { getCanonicalQuery } from "./getCanonicalQuery";
+export { getPayloadHash } from "./getPayloadHash";
+export { moveHeadersToQuery } from "./moveHeadersToQuery";
+export { prepareRequest } from "./prepareRequest";
+export { normalizeCredentialsProvider, normalizeRegionProvider } from "./normalizeProvider";
 export * from "./SignatureV4";

packages/signature-v4/src/index.ts

@@ -3,12 +3,12 @@ import { HttpRequest, QueryParameterBag } from "@aws-sdk/types";
 import { cloneRequest } from "./cloneRequest";
 
 /**
- * @internal
+ * @private
  */
-export function moveHeadersToQuery(
+export const moveHeadersToQuery = (
   request: HttpRequest,
   options: { unhoistableHeaders?: Set<string> } = {}
-): HttpRequest & { query: QueryParameterBag } {
+): HttpRequest & { query: QueryParameterBag } => {
   const { headers, query = {} as QueryParameterBag } =
     typeof (request as any).clone === "function" ? (request as any).clone() : cloneRequest(request);
   for (const name of Object.keys(headers)) {
@@ -24,4 +24,4 @@ export function moveHeadersToQuery(
     headers,
     query,
   };
-}
+};

packages/signature-v4/src/moveHeadersToQuery.ts

@@ -0,0 +1,27 @@
+import { Credentials, Provider } from "@aws-sdk/types";
+
+/**
+ * @private
+ */
+export const normalizeRegionProvider = (region: string | Provider<string>): Provider<string> => {
+  if (typeof region === "string") {
+    const promisified = Promise.resolve(region);
+    return () => promisified;
+  } else {
+    return region;
+  }
+};
+
+/**
+ * @private
+ */
+export const normalizeCredentialsProvider = (
+  credentials: Credentials | Provider<Credentials>
+): Provider<Credentials> => {
+  if (typeof credentials === "object") {
+    const promisified = Promise.resolve(credentials);
+    return () => promisified;
+  } else {
+    return credentials;
+  }
+};

packages/signature-v4/src/normalizeProvider.ts

@@ -4,9 +4,9 @@ import { cloneRequest } from "./cloneRequest";
 import { GENERATED_HEADERS } from "./constants";
 
 /**
- * @internal
+ * @private
  */
-export function prepareRequest(request: HttpRequest): HttpRequest {
+export const prepareRequest = (request: HttpRequest): HttpRequest => {
   // Create a clone of the request object that does not clone the body
   request = typeof (request as any).clone === "function" ? (request as any).clone() : cloneRequest(request);
 
@@ -17,4 +17,4 @@ export function prepareRequest(request: HttpRequest): HttpRequest {
   }
 
   return request;
-}
+};

packages/signature-v4/src/prepareRequest.ts

@@ -1,10 +1,9 @@
-export function iso8601(time: number | string | Date): string {
-  return toDate(time)
+export const iso8601 = (time: number | string | Date): string =>
+  toDate(time)
     .toISOString()
     .replace(/\.\d{3}Z$/, "Z");
-}
 
-export function toDate(time: number | string | Date): Date {
+export const toDate = (time: number | string | Date): Date => {
   if (typeof time === "number") {
     return new Date(time * 1000);
   }
@@ -17,4 +16,4 @@ export function toDate(time: number | string | Date): Date {
   }
 
   return time;
-}
+};

packages/signature-v4/src/utilDate.ts

@@ -4,7 +4,6 @@
     "declarationDir": "./dist/types",
     "rootDir": "./src",
     "outDir": "./dist/cjs",
-    "noUnusedLocals": true,
     "baseUrl": "."
   },
   "extends": "../../tsconfig.cjs.json",