feat(client-s3): support generating endpoints from multi-region access point (#2796)

* Revert "fix(client-s3): revert MRAP customizations (#2759)"

This reverts commit cfb3fff.

* chore: update mrap packages dependencies

* feat(middleware-sdk-s3): dynamically import aws-crt package optional dependencies

* feat(middleware-user-agent): track if aws-crt is available at runtime

* chore: update crt signer loading error message

* chore(middleware-sdk-s3): add crt package to dev dependency and track usage

This makes sure the type of CRT package is available and the Lerna builds
packages in right order--build crt signer package before buiding s3 middleware
package.

* docs(client-s3): add docs to Bucket members on using MRAP

* chore(signature-v4-crt): update crt version to ^1.9.7

* fix(client-s3): address PR feedbacks

Author: AllanZhengYP

Diff for: clients/client-s3/models/models_0.ts

@@ -514,6 +514,8 @@ export interface RestoreObjectRequest {
    * <p>The bucket name containing the object to restore. </p>
    *          <p>When using this action with an access point, you must direct requests to the access point hostname. The access point hostname takes the form <i>AccessPointName</i>-<i>AccountId</i>.s3-accesspoint.<i>Region</i>.amazonaws.com. When using this action with an access point through the Amazon Web Services SDKs, you provide the access point ARN in place of the bucket name. For more information about access point ARNs, see <a href="https://docs.aws.amazon.com/AmazonS3/latest/userguide/using-access-points.html">Using access points</a> in the <i>Amazon S3 User Guide</i>.</p>
    *          <p>When using this action with Amazon S3 on Outposts, you must direct requests to the S3 on Outposts hostname. The S3 on Outposts hostname takes the form <i>AccessPointName</i>-<i>AccountId</i>.<i>outpostID</i>.s3-outposts.<i>Region</i>.amazonaws.com. When using this action using S3 on Outposts through the Amazon Web Services SDKs, you provide the Outposts bucket ARN in place of the bucket name. For more information about S3 on Outposts ARNs, see <a href="https://docs.aws.amazon.com/AmazonS3/latest/userguide/S3onOutposts.html">Using S3 on Outposts</a> in the <i>Amazon S3 User Guide</i>.</p>
+   * <p>Note: To supply the Multi-region Access Point (MRAP) to Bucket, you need to install the "@aws-sdk/signature-v4-crt" package to your project dependencies.
+   * For more information, please go to https://github.com/aws/aws-sdk-js-v3#known-issues</p>
    */
   Bucket: string | undefined;
 
@@ -898,6 +900,8 @@ export namespace ScanRange {
 export interface SelectObjectContentRequest {
   /**
    * <p>The S3 bucket.</p>
+   * <p>Note: To supply the Multi-region Access Point (MRAP) to Bucket, you need to install the "@aws-sdk/signature-v4-crt" package to your project dependencies.
+   * For more information, please go to https://github.com/aws/aws-sdk-js-v3#known-issues</p>
    */
   Bucket: string | undefined;
 
@@ -1055,6 +1059,8 @@ export interface UploadPartRequest {
    * <p>The name of the bucket to which the multipart upload was initiated.</p>
    *          <p>When using this action with an access point, you must direct requests to the access point hostname. The access point hostname takes the form <i>AccessPointName</i>-<i>AccountId</i>.s3-accesspoint.<i>Region</i>.amazonaws.com. When using this action with an access point through the Amazon Web Services SDKs, you provide the access point ARN in place of the bucket name. For more information about access point ARNs, see <a href="https://docs.aws.amazon.com/AmazonS3/latest/userguide/using-access-points.html">Using access points</a> in the <i>Amazon S3 User Guide</i>.</p>
    *          <p>When using this action with Amazon S3 on Outposts, you must direct requests to the S3 on Outposts hostname. The S3 on Outposts hostname takes the form <i>AccessPointName</i>-<i>AccountId</i>.<i>outpostID</i>.s3-outposts.<i>Region</i>.amazonaws.com. When using this action using S3 on Outposts through the Amazon Web Services SDKs, you provide the Outposts bucket ARN in place of the bucket name. For more information about S3 on Outposts ARNs, see <a href="https://docs.aws.amazon.com/AmazonS3/latest/userguide/S3onOutposts.html">Using S3 on Outposts</a> in the <i>Amazon S3 User Guide</i>.</p>
+   * <p>Note: To supply the Multi-region Access Point (MRAP) to Bucket, you need to install the "@aws-sdk/signature-v4-crt" package to your project dependencies.
+   * For more information, please go to https://github.com/aws/aws-sdk-js-v3#known-issues</p>
    */
   Bucket: string | undefined;
 
@@ -1221,6 +1227,8 @@ export interface UploadPartCopyRequest {
    * <p>The bucket name.</p>
    *          <p>When using this action with an access point, you must direct requests to the access point hostname. The access point hostname takes the form <i>AccessPointName</i>-<i>AccountId</i>.s3-accesspoint.<i>Region</i>.amazonaws.com. When using this action with an access point through the Amazon Web Services SDKs, you provide the access point ARN in place of the bucket name. For more information about access point ARNs, see <a href="https://docs.aws.amazon.com/AmazonS3/latest/userguide/using-access-points.html">Using access points</a> in the <i>Amazon S3 User Guide</i>.</p>
    *          <p>When using this action with Amazon S3 on Outposts, you must direct requests to the S3 on Outposts hostname. The S3 on Outposts hostname takes the form <i>AccessPointName</i>-<i>AccountId</i>.<i>outpostID</i>.s3-outposts.<i>Region</i>.amazonaws.com. When using this action using S3 on Outposts through the Amazon Web Services SDKs, you provide the Outposts bucket ARN in place of the bucket name. For more information about S3 on Outposts ARNs, see <a href="https://docs.aws.amazon.com/AmazonS3/latest/userguide/S3onOutposts.html">Using S3 on Outposts</a> in the <i>Amazon S3 User Guide</i>.</p>
+   * <p>Note: To supply the Multi-region Access Point (MRAP) to Bucket, you need to install the "@aws-sdk/signature-v4-crt" package to your project dependencies.
+   * For more information, please go to https://github.com/aws/aws-sdk-js-v3#known-issues</p>
    */
   Bucket: string | undefined;
 

Diff for: clients/client-s3/models/models_1.ts

@@ -1,4 +1,5 @@
 import { defaultRegionInfoProvider } from "./endpoints";
+import { S3SignatureV4 } from "@aws-sdk/middleware-sdk-s3";
 import { Logger as __Logger } from "@aws-sdk/types";
 import { parseUrl } from "@aws-sdk/url-parser";
 import { S3ClientConfig } from "./S3Client";
@@ -12,6 +13,7 @@ export const getRuntimeConfig = (config: S3ClientConfig) => ({
   logger: config?.logger ?? ({} as __Logger),
   regionInfoProvider: config?.regionInfoProvider ?? defaultRegionInfoProvider,
   serviceId: config?.serviceId ?? "S3",
+  signerConstructor: config?.signerConstructor ?? S3SignatureV4,
   signingEscapePath: config?.signingEscapePath ?? false,
   urlParser: config?.urlParser ?? parseUrl,
   useArnRegion: config?.useArnRegion ?? false,

Diff for: clients/client-s3/runtimeConfig.shared.ts

@@ -19,16 +19,24 @@
 import static software.amazon.smithy.typescript.codegen.integration.RuntimeClientPlugin.Convention.HAS_MIDDLEWARE;
 
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.function.Consumer;
+import java.util.logging.Logger;
 import software.amazon.smithy.aws.traits.ServiceTrait;
+import software.amazon.smithy.build.PluginContext;
 import software.amazon.smithy.codegen.core.SymbolProvider;
 import software.amazon.smithy.model.Model;
 import software.amazon.smithy.model.knowledge.OperationIndex;
+import software.amazon.smithy.model.shapes.MemberShape;
 import software.amazon.smithy.model.shapes.OperationShape;
+import software.amazon.smithy.model.shapes.ServiceShape;
 import software.amazon.smithy.model.shapes.Shape;
+import software.amazon.smithy.model.shapes.ShapeId;
+import software.amazon.smithy.model.shapes.StructureShape;
+import software.amazon.smithy.model.traits.DocumentationTrait;
 import software.amazon.smithy.typescript.codegen.LanguageTarget;
 import software.amazon.smithy.typescript.codegen.TypeScriptDependency;
 import software.amazon.smithy.typescript.codegen.TypeScriptSettings;
@@ -45,6 +53,7 @@
  */
 @SmithyInternalApi
 public final class AddS3Config implements TypeScriptIntegration {
+    private static final Logger LOGGER = Logger.getLogger(AddS3Config.class.getName());
 
     private static final Set<String> SSEC_OPERATIONS = SetUtils.of("SSECustomerKey", "CopySourceSSECustomerKey");
 
@@ -60,6 +69,42 @@ public final class AddS3Config implements TypeScriptIntegration {
         "CompleteMultipartUpload"
     );
 
+    private static final String CRT_NOTIFICATION = "<p>Note: To supply the Multi-region Access Point (MRAP) to Bucket,"
+            + " you need to install the \"@aws-sdk/signature-v4-crt\" package to your project dependencies. \n"
+            + "For more information, please go to https://github.com/aws/aws-sdk-js-v3#known-issues</p>";
+
+    @Override
+    public Model preprocessModel(PluginContext context, TypeScriptSettings settings) {
+        Model model = context.getModel();
+        ServiceShape serviceShape = settings.getService(model);
+        if (!testServiceId(serviceShape)) {
+            return model;
+        }
+        Model.Builder modelBuilder = model.toBuilder();
+        Set<StructureShape> inputShapes = new HashSet<>();
+        for (ShapeId operationId : serviceShape.getAllOperations()) {
+            OperationShape operationShape = model.expectShape(operationId, OperationShape.class);
+            if (NON_BUCKET_ENDPOINT_OPERATIONS.contains(operationShape.getId().getName(serviceShape))) {
+                continue;
+            }
+            operationShape.getInput().ifPresent(inputShapeId -> {
+                StructureShape inputShape = model.expectShape(inputShapeId, StructureShape.class);
+                inputShape.getMember("Bucket").ifPresent(bucketMember -> {
+                    bucketMember.getTrait(DocumentationTrait.class).ifPresent(documentationTrait -> {
+                        StructureShape.Builder inputShapeBuilder = inputShape.toBuilder();
+                        MemberShape.Builder builder = MemberShape.shapeToBuilder(bucketMember);
+                        String newDocString = documentationTrait.getValue() + "\n" + CRT_NOTIFICATION;
+                        MemberShape newMemberShape = builder.addTrait(new DocumentationTrait(newDocString)).build();
+                        inputShapeBuilder.addMember(newMemberShape);
+                        inputShapes.add(inputShapeBuilder.build());
+                    });
+                });
+            });
+        }
+        LOGGER.info("Patching " + inputShapes.size() + " input shapes with CRT notification");
+        return modelBuilder.addShapes(inputShapes).build();
+    }
+
     @Override
     public void addConfigInterfaceFields(TypeScriptSettings settings, Model model, SymbolProvider symbolProvider,
             TypeScriptWriter writer) {
@@ -87,6 +132,10 @@ public Map<String, Consumer<TypeScriptWriter>> getRuntimeConfigWriters(TypeScrip
                     writer.write("false");
                 }, "useArnRegion", writer -> {
                     writer.write("false");
+                }, "signerConstructor", writer -> {
+                    writer.addDependency(AwsDependency.S3_MIDDLEWARE)
+                            .addImport("S3SignatureV4", "S3SignatureV4", AwsDependency.S3_MIDDLEWARE.packageName)
+                            .write("S3SignatureV4");
                 });
             case NODE:
                 return MapUtils.of("useArnRegion", writer -> {

Diff for: codegen/smithy-aws-typescript-codegen/src/main/java/software/amazon/smithy/aws/typescript/codegen/AddS3Config.java

@@ -134,6 +134,7 @@ describe("bucketEndpointMiddleware", () => {
         clientSigningRegion: mockRegion,
         useArnRegion: false,
         isCustomEndpoint: false,
+        disableMultiregionAccessPoints: false,
       });
       expect(previouslyResolvedConfig.region).toBeCalled();
       expect(previouslyResolvedConfig.regionInfoProvider).toBeCalled();

Diff for: packages/middleware-bucket-endpoint/src/bucketEndpointMiddleware.spec.ts

@@ -15,81 +15,76 @@ import { bucketHostname } from "./bucketHostname";
 import { getPseudoRegion } from "./bucketHostnameUtils";
 import { BucketEndpointResolvedConfig } from "./configurations";
 
-export const bucketEndpointMiddleware =
-  (options: BucketEndpointResolvedConfig): BuildMiddleware<any, any> =>
-  <Output extends MetadataBearer>(
-    next: BuildHandler<any, Output>,
-    context: HandlerExecutionContext
-  ): BuildHandler<any, Output> =>
-  async (args: BuildHandlerArguments<any>): Promise<BuildHandlerOutput<Output>> => {
-    const { Bucket: bucketName } = args.input as { Bucket: string };
-    let replaceBucketInPath = options.bucketEndpoint;
-    const request = args.request;
-    if (HttpRequest.isInstance(request)) {
-      if (options.bucketEndpoint) {
-        request.hostname = bucketName;
-      } else if (validateArn(bucketName)) {
-        const bucketArn = parseArn(bucketName);
-        const clientRegion = getPseudoRegion(await options.region());
-        const { partition, signingRegion = clientRegion } = (await options.regionInfoProvider(clientRegion)) || {};
-        const useArnRegion = await options.useArnRegion();
-        const {
-          hostname,
-          bucketEndpoint,
-          signingRegion: modifiedSigningRegion,
-          signingService,
-        } = bucketHostname({
-          bucketName: bucketArn,
-          baseHostname: request.hostname,
-          accelerateEndpoint: options.useAccelerateEndpoint,
-          dualstackEndpoint: options.useDualstackEndpoint,
-          pathStyleEndpoint: options.forcePathStyle,
-          tlsCompatible: request.protocol === "https:",
-          useArnRegion,
-          clientPartition: partition,
-          clientSigningRegion: signingRegion,
-          clientRegion: clientRegion,
-          isCustomEndpoint: options.isCustomEndpoint,
-        });
+export const bucketEndpointMiddleware = (options: BucketEndpointResolvedConfig): BuildMiddleware<any, any> => <
+  Output extends MetadataBearer
+>(
+  next: BuildHandler<any, Output>,
+  context: HandlerExecutionContext
+): BuildHandler<any, Output> => async (args: BuildHandlerArguments<any>): Promise<BuildHandlerOutput<Output>> => {
+  const { Bucket: bucketName } = args.input as { Bucket: string };
+  let replaceBucketInPath = options.bucketEndpoint;
+  const request = args.request;
+  if (HttpRequest.isInstance(request)) {
+    if (options.bucketEndpoint) {
+      request.hostname = bucketName;
+    } else if (validateArn(bucketName)) {
+      const bucketArn = parseArn(bucketName);
+      const clientRegion = getPseudoRegion(await options.region());
+      const { partition, signingRegion = clientRegion } = (await options.regionInfoProvider(clientRegion)) || {};
+      const useArnRegion = await options.useArnRegion();
+      const { hostname, bucketEndpoint, signingRegion: modifiedSigningRegion, signingService } = bucketHostname({
+        bucketName: bucketArn,
+        baseHostname: request.hostname,
+        accelerateEndpoint: options.useAccelerateEndpoint,
+        dualstackEndpoint: options.useDualstackEndpoint,
+        pathStyleEndpoint: options.forcePathStyle,
+        tlsCompatible: request.protocol === "https:",
+        useArnRegion,
+        clientPartition: partition,
+        clientSigningRegion: signingRegion,
+        clientRegion: clientRegion,
+        isCustomEndpoint: options.isCustomEndpoint,
+        disableMultiregionAccessPoints: await options.disableMultiregionAccessPoints(),
+      });
 
-        // If the request needs to use a region or service name inferred from ARN that different from client region, we
-        // need to set them in the handler context so the signer will use them
-        if (modifiedSigningRegion && modifiedSigningRegion !== signingRegion) {
-          context["signing_region"] = modifiedSigningRegion;
-        }
-        if (signingService && signingService !== "s3") {
-          context["signing_service"] = signingService;
-        }
+      // If the request needs to use a region or service name inferred from ARN that different from client region, we
+      // need to set them in the handler context so the signer will use them
+      if (modifiedSigningRegion && modifiedSigningRegion !== signingRegion) {
+        context["signing_region"] = modifiedSigningRegion;
+      }
+      if (signingService && signingService !== "s3") {
+        context["signing_service"] = signingService;
+      }
 
-        request.hostname = hostname;
-        replaceBucketInPath = bucketEndpoint;
-      } else {
-        const clientRegion = getPseudoRegion(await options.region());
-        const { hostname, bucketEndpoint } = bucketHostname({
-          bucketName,
-          clientRegion,
-          baseHostname: request.hostname,
-          accelerateEndpoint: options.useAccelerateEndpoint,
-          dualstackEndpoint: options.useDualstackEndpoint,
-          pathStyleEndpoint: options.forcePathStyle,
-          tlsCompatible: request.protocol === "https:",
-          isCustomEndpoint: options.isCustomEndpoint,
-        });
+      request.hostname = hostname;
+      replaceBucketInPath = bucketEndpoint;
+    } else {
+      const clientRegion = getPseudoRegion(await options.region());
+      const { hostname, bucketEndpoint } = bucketHostname({
+        bucketName,
+        clientRegion,
+        baseHostname: request.hostname,
+        accelerateEndpoint: options.useAccelerateEndpoint,
+        dualstackEndpoint: options.useDualstackEndpoint,
+        pathStyleEndpoint: options.forcePathStyle,
+        tlsCompatible: request.protocol === "https:",
+        isCustomEndpoint: options.isCustomEndpoint,
+      });
 
-        request.hostname = hostname;
-        replaceBucketInPath = bucketEndpoint;
-      }
+      request.hostname = hostname;
+      replaceBucketInPath = bucketEndpoint;
+    }
 
-      if (replaceBucketInPath) {
-        request.path = request.path.replace(/^(\/)?[^\/]+/, "");
-        if (request.path === "") {
-          request.path = "/";
-        }
+    if (replaceBucketInPath) {
+      request.path = request.path.replace(/^(\/)?[^\/]+/, "");
+      if (request.path === "") {
+        request.path = "/";
       }
     }
+  }
 
-    return next({ ...args, request });
-  };
+  return next({ ...args, request });
+};
 
 export const bucketEndpointMiddlewareOptions: RelativeMiddlewareOptions = {
   tags: ["BUCKET_ENDPOINT"],